@jrpool/kilotest 37.1.0 → 38.1.0

package/mcp.js

@@ -6,12 +6,14 @@
 // IMPORTS
 
 const {McpServer} = require('@modelcontextprotocol/sdk/server/mcp.js');
-const {StreamableHTTPServerTransport} = require('@modelcontextprotocol/sdk/server/streamableHttp.js');
+const {StreamableHTTPServerTransport} = require(
+  '@modelcontextprotocol/sdk/server/streamableHttp.js'
+);
 const {z} = require('zod');
 const {isReportAvailable, isURL} = require('./util');
 const targetAPI = require('./target/api');
-const reportListAPI = require('./reportList/api');
-const reportIssuesAPI = require('./reportIssues/api');
+const reportListAPI = require('./api/reportList');
+const reportSummaryAPI = require('./api/reportSummary');
 const reportIssueAPI = require('./reportIssue/api');
 const testRecFormAPI = require('./testRecForm/api');
 
@@ -29,7 +31,7 @@ const createMCPServer = () => {
     {
       description: 'Returns summary data from every available Kilotest report about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages that match the description or hostname fragment of a web page that you have provided. Matching is case-insensitive and succeeds if the page property either is included by or includes the specified fragment.',
       inputSchema: {
-        what: z.string().describe('All or part of a description of the web page.'),
+        description: z.string().describe('All or part of a description of the web page.'),
         hostname: z.string().describe('All or part of the hostname of the URL of the web page.')
       },
       annotations: {
@@ -40,18 +42,18 @@ const createMCPServer = () => {
         openWorldHint: false
       }
     },
-    async ({what, hostname}) => {
-      const result = await targetAPI.response([what, hostname]);
+    async ({description, hostname}) => {
+      const result = await targetAPI.response([description, hostname]);
       return {content: [{type: 'text', text: JSON.stringify(result)}]};
     }
   );
   server.registerTool(
-    'getListOfAllAvailableReports',
+    'listAllAvailableReports',
     {
       description: 'Returns a list of all available Kilotest reports; each report describes the results of a job that tested one web page for front-end quality (i.e. accessibility, usability, and standards conformity).',
       inputSchema: {},
       annotations: {
-        title: 'Get a list of all available reports',
+        title: 'List all available reports',
         readOnlyHint: true,
         idempotentHint: true,
         destructiveHint: false,
@@ -64,15 +66,15 @@ const createMCPServer = () => {
     }
   );
   server.registerTool(
-    'describeQualityOfOneWebPage',
+    'summarizeOneReport',
     {
-      description: 'Returns data from a specified Kilotest report about issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required timeStamp and jobID parameters identify the report and are obtained from a getListOfAllAvailableReports response.',
+      description: 'Returns a summary of a specified Kilotest report about the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required timeStamp and jobID parameters identify the report and are obtained from a listAllAvailableReports response.',
       inputSchema: {
-        timeStamp: z.string().describe('Report timestamp in YYMMDDTHHMM format, e.g. 260503T0432'),
+        timeStamp: z.string().describe('Timestamp of the report in YYMMDDTHHMM format, e.g. 260503T0432'),
         jobID: z.string().describe('Job identifier, e.g. x9z')
       },
       annotations: {
-        title: 'Describe the quality of one web page',
+        title: 'Summarize one report',
         readOnlyHint: true,
         idempotentHint: true,
         destructiveHint: false,
@@ -80,14 +82,14 @@ const createMCPServer = () => {
       }
     },
     async ({timeStamp, jobID}) => {
-      const result = await reportIssuesAPI.response([timeStamp, jobID]);
+      const result = await reportSummaryAPI.response([timeStamp, jobID]);
       return {content: [{type: 'text', text: JSON.stringify(result)}]};
     }
   );
   server.registerTool(
     'describeOneIssueFromOneReport',
     {
-      description: 'Returns data from a specified Kilotest report about one of the issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required issueID, timeStamp, and jobID parameters identify the issue and the report and are obtained from a describeQualityOfOneWebPage response.',
+      description: 'Returns data from a specified Kilotest report about one of the issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required issueID, timeStamp, and jobID parameters identify the issue and the report and are obtained from a summarizeOneReport response.',
       inputSchema: {
         issueID: z.string().describe('Issue identifier, e.g. contrastPoor'),
         timeStamp: z.string().describe('Report timestamp in YYMMDDTHHMM format, e.g. 260503T0432'),
@@ -109,11 +111,11 @@ const createMCPServer = () => {
   server.registerTool(
     'recommendQualityTestingOfOneWebPage',
     {
-      description: 'Recommends a web page for Kilotest to test for front-end quality (i.e. accessibility, usability, and standards conformity). Do not call this tool until after you call getListOfAllAvailableReports to check whether a report about the page, or a related page that satisfies your requirements, is available and to understand the naming conventions for pages.',
+      description: 'Recommends a web page for Kilotest to test for front-end quality (i.e. accessibility, usability, and standards conformity). Do not call this tool until after you call listAllAvailableReports to check whether a report about the page, or a related page that satisfies your requirements, is available and to understand the naming conventions for pages.',
       inputSchema: {
-        what: z.string().describe('Short description of the page, following the naming conventions visible in the getListOfAllAvailableReports response'),
-        url: z.string().describe('Full HTTPS URL of the page to test'),
-        why: z.string().describe('Reason for recommending this page for testing')
+        'description of the web page': z.string().describe('Short description of the page, following the naming conventions visible in the listAllAvailableReports response'),
+        'URL of the web page': z.string().describe('Full HTTPS URL of the page to test'),
+        'reason for testing the web page': z.string().describe('Reason for recommending this page for testing')
       },
       annotations: {
         title: 'Recommend quality testing of one web page',
@@ -123,7 +125,11 @@ const createMCPServer = () => {
         openWorldHint: false
       }
     },
-    async ({what, url, why}) => {
+    async ({
+      'description of the web page': what,
+      'URL of the web page': url,
+      'reason for testing the web page': why
+    }) => {
       if (!isURL(url)) {
         return {content: [{type: 'text', text: JSON.stringify({error: 'Invalid URL'})}], isError: true};
       }

package/openapi.yaml

@@ -35,9 +35,9 @@ paths:
 
   /api/reportList:
     get:
-      operationId: getListOfAllAvailableReports
-      summary: Get a list of all available reports
-      description: Get a list of all available reports. Each report describes the results of a job that tested a web page for front-end quality (i.e. accessibility, usability, and standards conformity). For each report, the list identifies which page was tested and when and provides URLs to begin incremental retrieval of the test results.
+      operationId: listAllAvailableReports
+      summary: List all available reports
+      description: List all available reports. Each report describes the results of a job that tested a web page for front-end quality (i.e. accessibility, usability, and standards conformity). For each report, the list identifies which page was tested and when and provides URLs to begin incremental retrieval of the test results.
       responses:
         '200':
           description: List of all available reports
@@ -50,7 +50,7 @@ paths:
     post:
       operationId: recommendQualityTestingOfOneWebPage
       summary: Recommend quality testing of one web page
-      description: Submit a recommendation for Kilotest to test, for the first time, a particular web page for front-end quality (i.e. accessibility, usability, and standards conformity). Recommendations are typically approved and the testing completed within a day, whereupon the results can be found with the getListOfAllAvailableReports operation. Before submitting a recommendation, use the getListOfAllAvailableReports operation to ensure that no report about the page is available, and also to see the stylistic rules for the naming of pages. An attempt to recommend a page for testing will fail if a report about the page is already available.
+      description: Submit a recommendation for Kilotest to test, for the first time, a particular web page for front-end quality (i.e. accessibility, usability, and standards conformity). Recommendations are typically approved and the testing completed within a day, whereupon the results can be found with the listAllAvailableReports operation. Before submitting a recommendation, use the listAllAvailableReports operation to ensure that no report about the page is available, and also to see the stylistic rules for the naming of pages. An attempt to recommend a page for testing will fail if a report about the page is already available.
       requestBody:
         description: Test recommendation specifications
         required: true
@@ -66,11 +66,11 @@ paths:
               schema:
                 $ref: '#/components/schemas/TestRecFormResponse'
 
-  /api/reportIssues/{timeStamp}/{jobID}:
+  /api/reportSummary/{timeStamp}/{jobID}:
     get:
-      operationId: describeQualityOfOneWebPage
-      summary: Describe quality of one web page
-      description: Get data about the quality of a specified Kilotest report. The data about each issue include its priority, the rule engines that reported it, the number of HTML elements exhibiting it, and URLs for retrieving element-level detail. The timeStamp and jobID parameters identify the report and are available in the response from the getListOfAllAvailableReports operation.
+      operationId: summarizeOneReport
+      summary: Summarize one report
+      description: Summarize one report. The summary should briefly describe the testing job and the results, including the rule engines that tested the web page and the issues that were revealed by the reported rule violations, and provide URLs for getting details about issues. The timeStamp and jobID parameters identify the report and were obtained from the response to a listAllAvailableReports operation.
       parameters:
         - name: timeStamp
           in: path
@@ -91,11 +91,11 @@ paths:
               - xx0
       responses:
         '200':
-          description: Data on issues from the specified report
+          description: Summary of the specified report
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ReportIssuesResponse'
+                $ref: '#/components/schemas/ReportSummaryResponse'
         '404':
           description: No report with the specified identifier is available
 
@@ -108,7 +108,7 @@ paths:
         - name: issueID
           in: path
           required: true
-          description: Issue identifier (e.g., imageNoText). Available under "issues reported" > priority level > "identifier" in the describeQualityOfOneWebPage response.
+          description: Issue identifier (e.g., imageNoText). Available under "response content" > "issues revealed" > [item] > "identifier" in the summarizeOneReport response.
           schema:
             type: string
             examples:
@@ -153,7 +153,7 @@ components:
         tool name:
           type: string
           examples:
-            - getListOfAllAvailableReports
+            - listAllAvailableReports
         request:
           type: object
           properties:
@@ -177,15 +177,78 @@ components:
             URLs:
               type: object
               properties:
-                'URL of your request':
+                'for you':
                   type: string
-                'equivalent URL for humans':
+                'for humans':
                   type: string
             closest ancestor request:
               oneOf:
                 - type: 'null'
                 - type: object
                   properties:
+                    identifier:
+                      type: string
+                    description:
+                      type: string
+                    URLs:
+                      type: object
+                      properties:
+                        'for you':
+                          type: string
+                        'for humans':
+                          type: string
+        response metadata:
+          type: object
+          properties:
+            identifier:
+              type: string
+              description: Unique identifier for this response instance.
+            date and time:
+              type: string
+              description: UTC date and time when the response was generated, in ISO 8601 format.
+              format: date-time
+
+    SharedFacts:
+      type: object
+      properties:
+        summary:
+          type: string
+          description: Natural-language facts about the request, the response, and Kilotest.
+        tool collection name:
+          type: string
+          examples:
+            - Kilotest
+        tool name:
+          type: string
+          examples:
+            - listAllAvailableReports
+        request:
+          type: object
+          properties:
+            description:
+              type: string
+            method:
+              type: string
+              enum:
+                - GET
+                - POST
+            payload:
+              type: object
+              description: The request payload, if method is POST, or otherwise omitted.
+            URLs:
+              type: object
+              properties:
+                'for you':
+                  type: string
+                'for humans':
+                  type: string
+            closest ancestor request:
+              oneOf:
+                - type: 'null'
+                - type: object
+                  properties:
+                    'tool name':
+                      type: string
                     description:
                       type: string
                     URLs:
@@ -209,13 +272,13 @@ components:
     TestRecForm:
       type: object
       properties:
-        what:
+        description of the web page:
           type: string
           description: Description of the web page to be tested.
-        url:
+        URL of the web page:
           type: string
           description: URL of the web page to be tested.
-        why:
+        reason for testing the web page:
           type: string
           description: Reason for testing the page.
 
@@ -273,7 +336,7 @@ components:
         for you:
           type: string
           format: uri
-          description: URL for an agent to request the next tier of detail.
+          description: URL for an LLM to request the next tier of detail.
         for humans:
           type: string
           format: uri
@@ -343,7 +406,7 @@ components:
         whether a later report about the same page exists:
           type: boolean
           description: If true, a more recent report about this page is available. If you want the latest results, use that report instead.
-        URLs for incrementally getting facts from the report:
+        URLs for more details:
           $ref: '#/components/schemas/NextTierURLs'
         size of the report in bytes:
           type: integer
@@ -354,7 +417,7 @@ components:
     Target:
       type: object
       properties:
-        what:
+        description:
           type: string
           description: All or part of a description of the web page. If the page is the home page of an organization, the description is the name of the organization (e.g., “Earthjustice”). Otherwise, the description appends minimal differentiating words to the name of the organization (e.g., “Earthjustice Donate”), without the word “page”.
         hostname:
@@ -363,12 +426,12 @@ components:
 
     ReportListResponse:
       allOf:
-        - $ref: '#/components/schemas/CommonResponseFields'
+        - $ref: '#/components/schemas/SharedFacts'
         - type: object
           properties:
-            available reports:
+            response content:
               type: array
-              description: One entry per available Kilotest report, in alphabetical order by page description and, in case of multiple reports per page, in order of creation date and time. Reports are matched by page description, not page URL, so, if only the URL of a page has changed between reports, the reports are treated as reports about the same page.
+              description: One entry per available Kilotest report, in alphabetical order by page description and, in case of multiple reports per page, in order of creation date and time. Whether two reports are about the same page is determined by the page descriptions, not page URLs.
               items:
                 $ref: '#/components/schemas/ReportListItem'
 
@@ -440,59 +503,51 @@ components:
           items:
             $ref: '#/components/schemas/IssueEntry'
 
-    ReportIssuesResponse:
+    IssueSummary:
+      type: object
+      properties:
+        identifier:
+          type: string
+        summary:
+          type: string
+        impact on a user:
+          type: string
+        priority:
+          type: string
+          enum:
+            - lowest
+            - low
+            - high
+            - highest
+        URLs for more details:
+          $ref: '#/components/schemas/NextTierURLs'
+
+    ReportSummaryResponse:
       allOf:
-        - $ref: '#/components/schemas/CommonResponseFields'
+        - $ref: '#/components/schemas/SharedFacts'
         - type: object
           properties:
-            report:
-              type: object
-              properties:
-                identifier:
-                  type: string
-                creation date:
-                  type: string
-                  format: date-time
-                days since the creation date:
-                  type: integer
-            tested web page:
+            response content:
               type: object
               properties:
-                description:
-                  type: string
-                URL:
-                  type: string
-                  format: uri
-            rule engines that tried to test the page:
-              type: array
-              items:
-                $ref: '#/components/schemas/RuleEngineInfo'
-            rule engines that were unable to test the page:
-              type: array
-              items:
-                $ref: '#/components/schemas/RuleEngineFailure'
-            rule engines that reported issues:
-              $ref: '#/components/schemas/RuleEnginesSummary'
-            number of issues reported:
-              type: object
-              properties:
-                total:
+                rule engines that tried to test the page:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/RuleEngineInfo'
+                rule engines that could not test the page:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/RuleEngineFailure'
+                names of rule engines that reported rule violations:
+                  type: array
+                  items:
+                    type: string
+                number of elements reported as violators:
                   type: integer
-                by priority:
-                  type: object
-                  properties:
-                    highest priority:
-                      type: integer
-                    high priority:
-                      type: integer
-                    low priority:
-                      type: integer
-                    lowest priority:
-                      type: integer
-            number of HTML elements reported as exhibiting issues:
-              type: integer
-            issues reported:
-              $ref: '#/components/schemas/IssuesByPriority'
+                issues revealed:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/IssueSummary'
 
     HTMLElementDetail:
       type: object

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "37.1.0",
+  "version": "38.1.0",
   "description": "An ensemble testing service with a focus on accessibility",
   "main": "index.js",
   "scripts": {

package/reannotate/index.js

@@ -21,9 +21,17 @@ exports.answer = async authCode => {
     for (const targetData of targetsData) {
       const [timeStamp, jobID] = targetData.jobName.split('-');
       // Reannotate it.
-      await annotateReport(ruleIDs, timeStamp, jobID);
+      const annotationError = await annotateReport(ruleIDs, timeStamp, jobID);
+      // If this failed:
+      if (annotationError) {
+        // Return an error page.
+        return {
+          status: 'error',
+          message: annotationError
+        };
+      }
     }
-    // Get the answer page.
+    // If every annotation succeeded, get the answer page.
     let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
     // Return it.
     return {

package/reannotateForm/index.js

@@ -18,11 +18,18 @@ const populateQuery = async query => {
   const reClassified = {};
   // For each target:
   for (const targetLog of targetLogs) {
-    const {jobName} = targetLog;
+    const {jobName = '-'} = targetLog;
     // Get the latest report on it.
     const report = await getReport(... jobName.split('-'));
-    const {acts} = report;
-    // For each act in the report:
+    const {acts = [], error} = report;
+    // If this failed:
+    if (error) {
+      // Populate the query with the reason.
+      query.error = error;
+      // Stop populating the query.
+      return;
+    }
+    // Otherwise, i.e. if it succeeded, for each act in the report:
     acts.forEach(act => {
       const {result, type, which} = act;
       // If it is a test act with standard instances:
@@ -100,7 +107,15 @@ exports.answer = async () => {
   const query = {};
   // Create a query to replace placeholders.
   await populateQuery(query);
-  // Get the template.
+  // If the query reports an error:
+  if (query.error) {
+    // Return the error.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, i.e. if the query does not report an error, get the template.
   let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
   // Replace its placeholders.
   Object.keys(query).forEach(param => {

package/reportHideForm/index.js

@@ -22,7 +22,15 @@ exports.answer = async (_, search) => {
     if (authCode === process.env.AUTH_CODE) {
       // Get the log of the report.
       const log = await getLog(... jobName.split('-'));
-      // Add a hiddenness property to the log.
+      // If this failed:
+      if (log.error) {
+        // Return why.
+        return {
+          status: 'error',
+          message: log.error
+        }
+      }
+      // Otherwise, i.e. if it succeeded, add a hiddenness property to the log.
       log.hidden = true;
       // Save the updated log.
       await fs.writeFile(path.join(logsPath, `${jobName}.json`), getJSON(log));
@@ -37,13 +45,24 @@ exports.answer = async (_, search) => {
     }
   }
   const reportFileNames = await fs.readdir(reportsPath);
+  // Initialize an array of report specifications.
   const reportSpecs = [];
   // For each report:
   for (const reportFileName of reportFileNames) {
     const [timeStamp, jobID] = reportFileName.slice(0, -5).split('-');
     // Get its log.
     const log = await getLog(timeStamp, jobID);
+    // If this failed:
+    if (log.error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: log.error
+      }
+    }
+    // Otherwise, i.e. if it succeeded:
     const {what, hidden} = log;
+    // Add the report to the array.
     reportSpecs.push({
       what,
       timeStamp,

package/reportIssue/api.js

@@ -24,8 +24,16 @@ const getIssueFacts = async (timeStamp, jobID, issue) => {
   const wcagType = wcag.length === 3 ? 'guideline' : 'success criterion';
   // Get the report.
   const report = await getReport(timeStamp, jobID);
+  const {acts, catalog, error} = report;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
   const violatorIndexSet = new Set();
-  const {acts, catalog} = report;
   // For each act of the report:
   acts.forEach(act => {
     // If it is a test act:
@@ -56,7 +64,10 @@ const getIssueFacts = async (timeStamp, jobID, issue) => {
       'names': reporters.map(tool => tool.toolName)
     },
     'number of HTML elements reported as exhibiting the issue': violatorCount,
-    'HTML elements reported as exhibiting the issue': Array.from(violatorIndexSet).map(index => ({
+    'HTML elements reported as exhibiting the issue': Array
+    .from(violatorIndexSet)
+    .filter(index => catalog[index])
+    .map(index => ({
       identifier: String(index),
       'tag name': catalog[index].tagName,
       'id attribute': catalog[index].id,
@@ -82,11 +93,19 @@ exports.response = async args => {
   }
   // Otherwise, i.e. if the report is available, get data on the report and its issues.
   const data = await getData(timeStamp, jobID);
-  const {pageData, issuesData} = data;
+  const {error, issuesData, pageData} = data;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
   const {what, url, daysAgo} = pageData;
   const {issueCount, issues, preventions, reporterCount, reporters, violatorCount} = issuesData;
   let issue;
-  // Get the level of and the data on the issue.
+  // Otherwise, i.e. if it succeeded, get the level of and the data on the issue.
   const issueLevel = [4, 3, 2, 1].find(level => {
     issue = issues[level].find(issue => issue.issueID === issueID);
     return issue;
@@ -116,14 +135,14 @@ exports.response = async args => {
       },
       method: 'GET',
       URLs: {
-        'URL of your request': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
-        'equivalent URL for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
+        'for you': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
+        'for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
       },
       'closest ancestor request': {
-        identifier: 'describeQualityOfOneWebPage',
-        description: 'Describe the quality of one web page.',
+        identifier: 'summarizeOneReport',
+        description: 'Summarize one report.',
         URLs: {
-          'for you': `${thisHost}/api/reportIssues/${timeStamp}/${jobID}`,
+          'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
           'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
         }
       }