@jrpool/kilotest 37.1.0 → 38.0.0

package/openapi.yaml

@@ -35,9 +35,9 @@ paths:
 
   /api/reportList:
     get:
-      operationId: getListOfAllAvailableReports
-      summary: Get a list of all available reports
-      description: Get a list of all available reports. Each report describes the results of a job that tested a web page for front-end quality (i.e. accessibility, usability, and standards conformity). For each report, the list identifies which page was tested and when and provides URLs to begin incremental retrieval of the test results.
+      operationId: listAllAvailableReports
+      summary: List all available reports
+      description: List all available reports. Each report describes the results of a job that tested a web page for front-end quality (i.e. accessibility, usability, and standards conformity). For each report, the list identifies which page was tested and when and provides URLs to begin incremental retrieval of the test results.
       responses:
         '200':
           description: List of all available reports
@@ -50,7 +50,7 @@ paths:
     post:
       operationId: recommendQualityTestingOfOneWebPage
       summary: Recommend quality testing of one web page
-      description: Submit a recommendation for Kilotest to test, for the first time, a particular web page for front-end quality (i.e. accessibility, usability, and standards conformity). Recommendations are typically approved and the testing completed within a day, whereupon the results can be found with the getListOfAllAvailableReports operation. Before submitting a recommendation, use the getListOfAllAvailableReports operation to ensure that no report about the page is available, and also to see the stylistic rules for the naming of pages. An attempt to recommend a page for testing will fail if a report about the page is already available.
+      description: Submit a recommendation for Kilotest to test, for the first time, a particular web page for front-end quality (i.e. accessibility, usability, and standards conformity). Recommendations are typically approved and the testing completed within a day, whereupon the results can be found with the listAllAvailableReports operation. Before submitting a recommendation, use the listAllAvailableReports operation to ensure that no report about the page is available, and also to see the stylistic rules for the naming of pages. An attempt to recommend a page for testing will fail if a report about the page is already available.
       requestBody:
         description: Test recommendation specifications
         required: true
@@ -66,11 +66,11 @@ paths:
               schema:
                 $ref: '#/components/schemas/TestRecFormResponse'
 
-  /api/reportIssues/{timeStamp}/{jobID}:
+  /api/reportSummary/{timeStamp}/{jobID}:
     get:
-      operationId: describeQualityOfOneWebPage
-      summary: Describe quality of one web page
-      description: Get data about the quality of a specified Kilotest report. The data about each issue include its priority, the rule engines that reported it, the number of HTML elements exhibiting it, and URLs for retrieving element-level detail. The timeStamp and jobID parameters identify the report and are available in the response from the getListOfAllAvailableReports operation.
+      operationId: summarizeOneReport
+      summary: Summarize one report
+      description: Summarize one report. The summary should briefly describe the testing job and the results, including the rule engines that tested the web page and the issues that were revealed by the reported rule violations, and provide URLs for getting details about issues. The timeStamp and jobID parameters identify the report and were obtained from the response to a listAllAvailableReports operation.
       parameters:
         - name: timeStamp
           in: path
@@ -91,11 +91,11 @@ paths:
               - xx0
       responses:
         '200':
-          description: Data on issues from the specified report
+          description: Summary of the specified report
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ReportIssuesResponse'
+                $ref: '#/components/schemas/ReportSummaryResponse'
         '404':
           description: No report with the specified identifier is available
 
@@ -108,7 +108,7 @@ paths:
         - name: issueID
           in: path
           required: true
-          description: Issue identifier (e.g., imageNoText). Available under "issues reported" > priority level > "identifier" in the describeQualityOfOneWebPage response.
+          description: Issue identifier (e.g., imageNoText). Available under "response content" > "issues revealed" > [item] > "identifier" in the summarizeOneReport response.
           schema:
             type: string
             examples:
@@ -153,7 +153,7 @@ components:
         tool name:
           type: string
           examples:
-            - getListOfAllAvailableReports
+            - listAllAvailableReports
         request:
           type: object
           properties:
@@ -177,15 +177,78 @@ components:
             URLs:
               type: object
               properties:
-                'URL of your request':
+                'for you':
                   type: string
-                'equivalent URL for humans':
+                'for humans':
                   type: string
             closest ancestor request:
               oneOf:
                 - type: 'null'
                 - type: object
                   properties:
+                    identifier:
+                      type: string
+                    description:
+                      type: string
+                    URLs:
+                      type: object
+                      properties:
+                        'for you':
+                          type: string
+                        'for humans':
+                          type: string
+        response metadata:
+          type: object
+          properties:
+            identifier:
+              type: string
+              description: Unique identifier for this response instance.
+            date and time:
+              type: string
+              description: UTC date and time when the response was generated, in ISO 8601 format.
+              format: date-time
+
+    SharedFacts:
+      type: object
+      properties:
+        summary:
+          type: string
+          description: Natural-language facts about the request, the response, and Kilotest.
+        tool collection name:
+          type: string
+          examples:
+            - Kilotest
+        tool name:
+          type: string
+          examples:
+            - listAllAvailableReports
+        request:
+          type: object
+          properties:
+            description:
+              type: string
+            method:
+              type: string
+              enum:
+                - GET
+                - POST
+            payload:
+              type: object
+              description: The request payload, if method is POST, or otherwise omitted.
+            URLs:
+              type: object
+              properties:
+                'for you':
+                  type: string
+                'for humans':
+                  type: string
+            closest ancestor request:
+              oneOf:
+                - type: 'null'
+                - type: object
+                  properties:
+                    'tool name':
+                      type: string
                     description:
                       type: string
                     URLs:
@@ -209,13 +272,13 @@ components:
     TestRecForm:
       type: object
       properties:
-        what:
+        description of the web page:
           type: string
           description: Description of the web page to be tested.
-        url:
+        URL of the web page:
           type: string
           description: URL of the web page to be tested.
-        why:
+        reason for testing the web page:
           type: string
           description: Reason for testing the page.
 
@@ -273,7 +336,7 @@ components:
         for you:
           type: string
           format: uri
-          description: URL for an agent to request the next tier of detail.
+          description: URL for an LLM to request the next tier of detail.
         for humans:
           type: string
           format: uri
@@ -343,7 +406,7 @@ components:
         whether a later report about the same page exists:
           type: boolean
           description: If true, a more recent report about this page is available. If you want the latest results, use that report instead.
-        URLs for incrementally getting facts from the report:
+        URLs for more details:
           $ref: '#/components/schemas/NextTierURLs'
         size of the report in bytes:
           type: integer
@@ -354,7 +417,7 @@ components:
     Target:
       type: object
       properties:
-        what:
+        description:
           type: string
           description: All or part of a description of the web page. If the page is the home page of an organization, the description is the name of the organization (e.g., “Earthjustice”). Otherwise, the description appends minimal differentiating words to the name of the organization (e.g., “Earthjustice Donate”), without the word “page”.
         hostname:
@@ -363,12 +426,12 @@ components:
 
     ReportListResponse:
       allOf:
-        - $ref: '#/components/schemas/CommonResponseFields'
+        - $ref: '#/components/schemas/SharedFacts'
         - type: object
           properties:
-            available reports:
+            response content:
               type: array
-              description: One entry per available Kilotest report, in alphabetical order by page description and, in case of multiple reports per page, in order of creation date and time. Reports are matched by page description, not page URL, so, if only the URL of a page has changed between reports, the reports are treated as reports about the same page.
+              description: One entry per available Kilotest report, in alphabetical order by page description and, in case of multiple reports per page, in order of creation date and time. Whether two reports are about the same page is determined by the page descriptions, not page URLs.
               items:
                 $ref: '#/components/schemas/ReportListItem'
 
@@ -440,59 +503,51 @@ components:
           items:
             $ref: '#/components/schemas/IssueEntry'
 
-    ReportIssuesResponse:
+    IssueSummary:
+      type: object
+      properties:
+        identifier:
+          type: string
+        summary:
+          type: string
+        impact on a user:
+          type: string
+        priority:
+          type: string
+          enum:
+            - lowest
+            - low
+            - high
+            - highest
+        URLs for more details:
+          $ref: '#/components/schemas/NextTierURLs'
+
+    ReportSummaryResponse:
       allOf:
-        - $ref: '#/components/schemas/CommonResponseFields'
+        - $ref: '#/components/schemas/SharedFacts'
         - type: object
           properties:
-            report:
-              type: object
-              properties:
-                identifier:
-                  type: string
-                creation date:
-                  type: string
-                  format: date-time
-                days since the creation date:
-                  type: integer
-            tested web page:
+            response content:
               type: object
               properties:
-                description:
-                  type: string
-                URL:
-                  type: string
-                  format: uri
-            rule engines that tried to test the page:
-              type: array
-              items:
-                $ref: '#/components/schemas/RuleEngineInfo'
-            rule engines that were unable to test the page:
-              type: array
-              items:
-                $ref: '#/components/schemas/RuleEngineFailure'
-            rule engines that reported issues:
-              $ref: '#/components/schemas/RuleEnginesSummary'
-            number of issues reported:
-              type: object
-              properties:
-                total:
+                rule engines that tried to test the page:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/RuleEngineInfo'
+                rule engines that could not test the page:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/RuleEngineFailure'
+                names of rule engines that reported rule violations:
+                  type: array
+                  items:
+                    type: string
+                number of elements reported as violators:
                   type: integer
-                by priority:
-                  type: object
-                  properties:
-                    highest priority:
-                      type: integer
-                    high priority:
-                      type: integer
-                    low priority:
-                      type: integer
-                    lowest priority:
-                      type: integer
-            number of HTML elements reported as exhibiting issues:
-              type: integer
-            issues reported:
-              $ref: '#/components/schemas/IssuesByPriority'
+                issues revealed:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/IssueSummary'
 
     HTMLElementDetail:
       type: object

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "37.1.0",
+  "version": "38.0.0",
   "description": "An ensemble testing service with a focus on accessibility",
   "main": "index.js",
   "scripts": {

package/reannotate/index.js

@@ -21,9 +21,17 @@ exports.answer = async authCode => {
     for (const targetData of targetsData) {
       const [timeStamp, jobID] = targetData.jobName.split('-');
       // Reannotate it.
-      await annotateReport(ruleIDs, timeStamp, jobID);
+      const annotationError = await annotateReport(ruleIDs, timeStamp, jobID);
+      // If this failed:
+      if (annotationError) {
+        // Return an error page.
+        return {
+          status: 'error',
+          message: annotationError
+        };
+      }
     }
-    // Get the answer page.
+    // If every annotation succeeded, get the answer page.
     let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
     // Return it.
     return {

package/reannotateForm/index.js

@@ -18,11 +18,18 @@ const populateQuery = async query => {
   const reClassified = {};
   // For each target:
   for (const targetLog of targetLogs) {
-    const {jobName} = targetLog;
+    const {jobName = '-'} = targetLog;
     // Get the latest report on it.
     const report = await getReport(... jobName.split('-'));
-    const {acts} = report;
-    // For each act in the report:
+    const {acts = [], error} = report;
+    // If this failed:
+    if (error) {
+      // Populate the query with the reason.
+      query.error = error;
+      // Stop populating the query.
+      return;
+    }
+    // Otherwise, i.e. if it succeeded, for each act in the report:
     acts.forEach(act => {
       const {result, type, which} = act;
       // If it is a test act with standard instances:
@@ -100,7 +107,15 @@ exports.answer = async () => {
   const query = {};
   // Create a query to replace placeholders.
   await populateQuery(query);
-  // Get the template.
+  // If the query reports an error:
+  if (query.error) {
+    // Return the error.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, i.e. if the query does not report an error, get the template.
   let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
   // Replace its placeholders.
   Object.keys(query).forEach(param => {

package/reportHideForm/index.js

@@ -22,7 +22,15 @@ exports.answer = async (_, search) => {
     if (authCode === process.env.AUTH_CODE) {
       // Get the log of the report.
       const log = await getLog(... jobName.split('-'));
-      // Add a hiddenness property to the log.
+      // If this failed:
+      if (log.error) {
+        // Return why.
+        return {
+          status: 'error',
+          message: log.error
+        }
+      }
+      // Otherwise, i.e. if it succeeded, add a hiddenness property to the log.
       log.hidden = true;
       // Save the updated log.
       await fs.writeFile(path.join(logsPath, `${jobName}.json`), getJSON(log));
@@ -37,13 +45,24 @@ exports.answer = async (_, search) => {
     }
   }
   const reportFileNames = await fs.readdir(reportsPath);
+  // Initialize an array of report specifications.
   const reportSpecs = [];
   // For each report:
   for (const reportFileName of reportFileNames) {
     const [timeStamp, jobID] = reportFileName.slice(0, -5).split('-');
     // Get its log.
     const log = await getLog(timeStamp, jobID);
+    // If this failed:
+    if (log.error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: log.error
+      }
+    }
+    // Otherwise, i.e. if it succeeded:
     const {what, hidden} = log;
+    // Add the report to the array.
     reportSpecs.push({
       what,
       timeStamp,

package/reportIssue/api.js

@@ -24,8 +24,16 @@ const getIssueFacts = async (timeStamp, jobID, issue) => {
   const wcagType = wcag.length === 3 ? 'guideline' : 'success criterion';
   // Get the report.
   const report = await getReport(timeStamp, jobID);
+  const {acts, catalog, error} = report;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
   const violatorIndexSet = new Set();
-  const {acts, catalog} = report;
   // For each act of the report:
   acts.forEach(act => {
     // If it is a test act:
@@ -56,7 +64,10 @@ const getIssueFacts = async (timeStamp, jobID, issue) => {
       'names': reporters.map(tool => tool.toolName)
     },
     'number of HTML elements reported as exhibiting the issue': violatorCount,
-    'HTML elements reported as exhibiting the issue': Array.from(violatorIndexSet).map(index => ({
+    'HTML elements reported as exhibiting the issue': Array
+    .from(violatorIndexSet)
+    .filter(index => catalog[index])
+    .map(index => ({
       identifier: String(index),
       'tag name': catalog[index].tagName,
       'id attribute': catalog[index].id,
@@ -82,11 +93,19 @@ exports.response = async args => {
   }
   // Otherwise, i.e. if the report is available, get data on the report and its issues.
   const data = await getData(timeStamp, jobID);
-  const {pageData, issuesData} = data;
+  const {error, issuesData, pageData} = data;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
   const {what, url, daysAgo} = pageData;
   const {issueCount, issues, preventions, reporterCount, reporters, violatorCount} = issuesData;
   let issue;
-  // Get the level of and the data on the issue.
+  // Otherwise, i.e. if it succeeded, get the level of and the data on the issue.
   const issueLevel = [4, 3, 2, 1].find(level => {
     issue = issues[level].find(issue => issue.issueID === issueID);
     return issue;
@@ -116,14 +135,14 @@ exports.response = async args => {
       },
       method: 'GET',
       URLs: {
-        'URL of your request': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
-        'equivalent URL for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
+        'for you': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
+        'for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
       },
       'closest ancestor request': {
-        identifier: 'describeQualityOfOneWebPage',
-        description: 'Describe the quality of one web page.',
+        identifier: 'summarizeOneReport',
+        description: 'Summarize one report.',
         URLs: {
-          'for you': `${thisHost}/api/reportIssues/${timeStamp}/${jobID}`,
+          'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
           'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
         }
       }

package/reportIssue/index.js

@@ -1,6 +1,6 @@
 /*
   index.js
-  Answers the diagnoses question.
+  Answers the reportIssue question.
 */
 
 // IMPORTS
@@ -25,10 +25,26 @@ const path = require('path');
 
 // Adds parameters to a query for the answer page.
 const populateQuery = async (issueID, timeStamp, jobID, query) => {
-  // Add facts about the issue to the query.
-  query.issue = issues[issueID].summary;
+  // Get descriptions of the page facts.
   const pageDataStrings = await getPageDataStrings(timeStamp, jobID);
-  const {what, url, urlLink, testInfo} = pageDataStrings;
+  // If this failed:
+  if (pageDataStrings.error) {
+    // Populate the query with the reason.
+    query.error = pageDataStrings.error;
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, add the issue summary to the query.
+  query.issue = issues[issueID]?.summary;
+  // If adding the issue summary failed:
+  if (! query.issue) {
+    // Populate the query with the reason.
+    query.error = 'Issue not found';
+    // Stop populating the query.
+    return;
+  }
+  const {testInfo, url, urlLink, what} = pageDataStrings;
+  // Add page facts to the query.
   query.target = what;
   query.urlLink = urlLink;
   query.testInfo = testInfo;
@@ -44,6 +60,14 @@ const populateQuery = async (issueID, timeStamp, jobID, query) => {
   // Get the report.
   const report = await getReport(timeStamp, jobID);
   const {acts, catalog} = report;
+  // If this failed:
+  if (report.error) {
+    // Populate the query with the reason.
+    query.error = report.error;
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, get the test acts of the report.
   const testActs = acts.filter(act => act.type === 'test');
   // For each test act:
   testActs.forEach(act => {
@@ -159,7 +183,15 @@ exports.answer = async pageArgs => {
   const query = {};
   // Create a query to replace the placeholders.
   await populateQuery(issueID, timeStamp, jobID, query);
-  // If the test specifications are valid:
+  // If this failed:
+  if (query.error) {
+    // Return the error.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, if it succeeded and the report facts were obtained:
   if (query.testInfo) {
     // Get the template.
     let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
@@ -173,9 +205,9 @@ exports.answer = async pageArgs => {
       answerPage
     };
   }
-  // Otherwise, report this.
+  // Otherwise, i.e. if the report facts were not obtained, report this.
   return {
     status: 'error',
-    message: 'Invalid report specification'
+    message: 'Report facts not obtained'
   };
 };

package/reportIssues/index.js

@@ -24,20 +24,24 @@ const populateQuery = async (timeStamp, jobID, query) => {
   // Get data on the target and its issues according to the report.
   const data = await getData(timeStamp, jobID);
   const {pageData, issuesData} = data;
-  // If the page data are invalid:
-  if (typeof pageData === 'string') {
-    // Return this.
-    return pageData;
+  // If this failed:
+  if (data.error) {
+    // Populate the query with the reason.
+    query.error = data.error;
+    // Stop populating the query.
+    return;
   }
-  // Otherwise, if the issues data are invalid:
-  if (typeof issuesData === 'string') {
-    // Return this.
-    return issuesData;
-  }
-  // Otherwise, get fact descriptions for the target.
+  // Otherwise, i.e. if it succeeded, get fact descriptions for the target.
   const pageInfo = await getPageDataStrings(timeStamp, jobID, pageData);
-  const {what, urlLink, testInfo} = pageInfo;
-  // Add target data to the query.
+  const {testInfo, urlLink, what} = pageInfo;
+  // If this failed:
+  if (pageInfo.error) {
+    // Populate the query with the reason.
+    query.error = pageInfo.error;
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, add target data to the query.
   query.target = what;
   query.urlLink = urlLink;
   query.testInfo = testInfo;
@@ -154,7 +158,15 @@ exports.answer = async pageArgs => {
   const query = {};
   // Create a query to replace the placeholders.
   await populateQuery(timeStamp, jobID, query);
-  // If the report facts were obtained:
+  // If this failed:
+  if (query.error) {
+    // Return the error.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, if it succeeded and the report facts were obtained:
   if (query.testInfo) {
     // Get the template.
     let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
@@ -171,6 +183,6 @@ exports.answer = async pageArgs => {
   // Otherwise, i.e. if they were not obtained, report this.
   return {
     status: 'error',
-    message: 'Report processing failed'
+    message: 'Report facts not obtained'
   };
 };