@jrpool/kilotest 37.0.0 → 38.0.0

package/alerts.js

@@ -19,7 +19,7 @@ const ALERT_FROM = process.env.ALERT_FROM;
 // FUNCTIONS
 
 // Sends an email alert to a manager.
-exports.sendAlert = (subject, body) => new Promise((resolve, reject) => {
+exports.sendAlert = (subject, body) => new Promise(resolve => {
   // If the alert configuration is complete:
   if (MANAGER_EMAIL && ALERT_API_HOST && ALERT_API_PATH && ALERT_API_KEY && ALERT_FROM) {
     const payload = JSON.stringify({

package/diagnoses/index.js

@@ -23,10 +23,27 @@ const path = require('path');
 
 // Adds parameters to a query for the answer page.
 const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, query) => {
+  // Get descriptions of the page facts.
   const pageDataStrings = await getPageDataStrings(timeStamp, jobID);
-  const {what, url, urlLink, testInfo} = pageDataStrings;
+  // If this failed:
+  if (pageDataStrings.error) {
+    // Populate the query with the reason.
+    query.error = pageDataStrings.error;
+    // Stop populating the query.
+    return;
+  }
+  const {testInfo, url, urlLink, what} = pageDataStrings;
+  // Otherwise, i.e. if it succeeded, get the report.
   const report = await getReport(timeStamp, jobID);
   const {acts, catalog} = report;
+  // If this failed:
+  if (report.error) {
+    // Populate the query with the reason.
+    query.error = report.error;
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, get the catalog item of the specified violator.
   const catalogItem = catalog[catalogIndex] ?? {};
   const {boxID, startTag, tagName, text} = catalogItem;
   query.catalogIndex = catalogIndex;
@@ -45,7 +62,15 @@ const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, qu
   query.target = what;
   query.urlLink = urlLink;
   query.testInfo = testInfo;
-  query.issue = issues[issueID].summary;
+  query.issue = issues[issueID]?.summary;
+  // If adding the issue summary failed:
+  if (! query.issue) {
+    // Populate the query with the reason.
+    query.error = 'Issue not found';
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, get the issue details.
   const issue = issues[issueID];
   const {wcag, weight, why} = issue;
   query.why = why;
@@ -118,7 +143,15 @@ exports.answer = async (pageArgs, search) => {
   const query = {};
   // Create a query to replace the placeholders.
   await populateQuery(issueID, timeStamp, jobID, catalogIndex, pathID, query);
-  // If the test specifications are valid:
+  // If this failed:
+  if (query.error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, if it succeeded and the report facts were obtained:
   if (query.testInfo) {
     // Get the template.
     let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
@@ -132,9 +165,10 @@ exports.answer = async (pageArgs, search) => {
       answerPage
     };
   }
-  // Otherwise, report this.
+  // Otherwise, i.e. if the report facts were not obtained:
+  // Report this.
   return {
     status: 'error',
-    message: 'Invalid report specification'
+    message: 'Report facts not obtained'
   };
 };

package/eslint.config.mjs

@@ -3,7 +3,7 @@ import globals from "globals";
 import json from "@eslint/json";
 import markdown from "@eslint/markdown";
 import css from "@eslint/css";
-import { defineConfig } from "eslint/config";
+import {defineConfig} from "eslint/config";
 
 export default defineConfig([
   {
@@ -18,6 +18,9 @@ export default defineConfig([
     extends: ["js/recommended"],
     languageOptions: {
       globals: globals.node
+    },
+    rules: {
+      "no-control-regex": "off"
     }
   },
   {

package/index.js

@@ -30,6 +30,7 @@ const {
 } = require('./util');
 const {handleMCP, mcpPath} = require('./mcp');
 const fs = require('fs/promises');
+const {handleComment} = require('./tutorial/index');
 const http = require('http');
 const https = require('https');
 const path = require('path');
@@ -273,26 +274,30 @@ const requestHandler = async (request, response) => {
       if (isTimeStamp(timeStamp) && isJobID(jobID)) {
         const reportHidden = await isHidden(timeStamp, jobID);
         // If the report exists and is hidden:
-        if (reportHidden === true) {
-          console.log(`ERROR: Hidden report ${timeStamp}-${jobID} requested`);
+        if (reportHidden) {
+          console.error(`Hidden report ${timeStamp}-${jobID} requested`);
           // Report this as suspected abuse.
           await serveError(
-            getAbuseError(request, `Requested report ${timeStamp}-${jobID} not available`),
+            getAbuseError(request, `Hidden report ${timeStamp}-${jobID} requested`),
             response,
             true
           );
         }
-        // Otherwise, if the report does not exist or another error occurred:
-        else if (typeof reportHidden === 'string') {
-          // Report this as suspected abuse.
-          await serveError(getAbuseError(request, reportHidden), response, true);
-        }
-        // Otherwise, i.e. if the report log is valid and not hidden:
+        // Otherwise, i.e. if the report is not hidden:
         else {
-          // Get the report.
+          // Get it.
           const report = await getReport(timeStamp, jobID);
-          // If it exists and is valid:
-          if (typeof report === 'object') {
+          // If this failed:
+          if (report.error) {
+            // Report this as suspected abuse.
+            await serveError(
+              getAbuseError(request, `Nonexistent report ${timeStamp}-${jobID} requested`),
+              response,
+              true
+            );
+          }
+          // Otherwise, i.e. if it succeeded:
+          else {
             // Serve response headers for a JSON download.
             setHeaders('application/json', null, 'low');
             response.setHeader(
@@ -301,11 +306,6 @@ const requestHandler = async (request, response) => {
             // Download the report.
             response.end(getJSON(report));
           }
-          // Otherwise, i.e. if an error occurred:
-          else {
-            // Report it.
-            await serveError({message: report}, response, true);
-          }
         }
       }
       // Otherwise, i.e. if the request is syntactically invalid:
@@ -350,10 +350,10 @@ const requestHandler = async (request, response) => {
         setHeaders('application/json', null, 'ultra');
         response.end(JSON.stringify(responseData));
       }
-      // Otherwise, if the service is provision of issue statistics for a report:
-      else if (service === 'reportIssues') {
+      // Otherwise, if the service is provision of a report summary:
+      else if (service === 'reportSummary') {
         // Get the response (potentially error) data.
-        const responseData = await require(path.join(__dirname, 'reportIssues', 'api'))
+        const responseData = await require(path.join(__dirname, 'reportSummary', 'api'))
         .response(specs);
         // Send them.
         setHeaders('application/json', null, 'high');
@@ -701,12 +701,12 @@ const requestHandler = async (request, response) => {
         }
         // Otherwise, if the first segment is the report finding service:
         else if (segments[0] === 'target') {
-          const {what = '', hostname = ''} = postData;
-          // If the payload contains a description fragment or hostname fragment:
-          if (what || hostname) {
+          const {description = '', hostname = ''} = postData;
+          // If the payload contains a description or hostname:
+          if (description || hostname) {
             // Process the request and get the response data.
             const responseData = await require(path.join(__dirname, 'target', 'api'))
-            .response([what, hostname]);
+            .response([description, hostname]);
             // Send them.
             setHeaders('application/json', null, 'ultra');
             response.end(JSON.stringify(responseData));
@@ -731,7 +731,8 @@ const requestHandler = async (request, response) => {
             if (await isReportAvailable(what, url)) {
               // Report this.
               responseData = {
-                warning: 'A report on the page is already available'
+                status: 'warning',
+                message: 'A report on the page is already available'
               }
             }
             // Otherwise, i.e. if no report on the page is available:
@@ -762,7 +763,7 @@ const requestHandler = async (request, response) => {
       else if (pageName === 'tutorialComment.html') {
         const {content} = postData;
         setHeaders('application/json', null, 'low');
-        const answerData = await require(path.join(__dirname, 'tutorial', 'index')).saveComment(content);
+        const answerData = await handleComment(content);
         if (answerData.status === 'ok') {
           response.end(JSON.stringify({status: 'ok'}));
         }

package/issues/index.js

@@ -1,6 +1,6 @@
 /*
   index.js
-  Answers the targets question.
+  Answers the issues question.
 */
 
 // IMPORTS
@@ -24,6 +24,8 @@ const path = require('path');
 
 // Gets summary data on the issues reported in a set of reports.
 const getIssuesSummary = async logs => {
+  // Initialize the summary.
+  const summary = {};
   // Initialize data for a summary.
   const issuesData = {};
   // For each log of a report to be inspected:
@@ -37,8 +39,14 @@ const getIssuesSummary = async logs => {
     }
     // Get the corresponding report.
     const report = await getReport(timeStamp, jobID);
-    // For each act in it:
-    report.acts.forEach(act => {
+    const {acts = [], error} = report;
+    // If this failed:
+    if (error) {
+      // Populate the summary with the reason.
+      summary.error = error;
+    }
+    // For each act in it (none if the report retrieval failed):
+    acts.forEach(act => {
       // If it is a test act:
       if (act.type === 'test') {
         const {result, which} = act;
@@ -60,11 +68,9 @@ const getIssuesSummary = async logs => {
       }
     });
   }
-  // Initialize the summary.
-  const summary = {
-    totalCount: 0,
-    issues: []
-  };
+  // Initialize the summary properties.
+  summary.totalCount = 0;
+  summary.issues = [];
   // For each issue:
   Object.entries(issuesData).forEach(([issueID, data]) => {
     const {count, reporters} = data;
@@ -107,7 +113,14 @@ const populateQuery = async query => {
   const targetLogs = (await getLogs()).filter(log => ! log.superseded);
   // Get summary data on the issues.
   const issuesSummary = await getIssuesSummary(targetLogs);
-  // Initialize the lines.
+  // If this failed:
+  if (issuesSummary.error) {
+    // Populate the query with the reason.
+    query.error = issuesSummary.error;
+    // Stop populating the query.
+    return;
+  }
+  // Otherwise, i.e. if it succeeded, initialize the lines.
   const lines = [];
   const margin = ' '.repeat(6);
   // For each weight:
@@ -159,7 +172,15 @@ exports.answer = async () => {
   const query = {};
   // Create a query to replace placeholders.
   await populateQuery(query);
-  // Get the template.
+  // If the query reports an error:
+  if (query.error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: query.error
+    };
+  }
+  // Otherwise, i.e. if the query does not report an error, get the template.
   let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
   // Replace its placeholders.
   Object.keys(query).forEach(param => {

package/llms-full.txt

@@ -63,7 +63,7 @@ Kilotest offers a comprehensive suite of capabilities to users via its web UI:
 - Get statistics about issues reported in one report: `https://kilotest.com/reportIssues.html/{timeStamp}/{jobID}`
 - Get details about one issue reported in one report: `https://kilotest.com/reportIssue.html/{issueID}/{timeStamp}/{jobID}`
 - Get diagnoses by rule engines of rule violations for one HTML element exhibiting one issue in one report: `https://kilotest.com/diagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
-- [Recommend the testing of a not-yet-tested page](`https://kilotest.com/testRecForm.html`)
+- [Recommend the testing of a not-yet-tested page](https://kilotest.com/testRecForm.html)
 - Recommend the retesting of a previously tested page: `https://kilotest.com/retestRecForm.html/{timeStamp}/{jobID}`
 - [Get statistics about frequently reported issues across all pages](https://kilotest.com/issues.html)
 - [Get a list of the operations that Kilotest managers are authorized to request](https://kilotest.com/manage.html)
@@ -73,28 +73,28 @@ Kilotest offers a comprehensive suite of capabilities to users via its web UI:
 
 Kilotest is implementing a richer suite of capabilities optimized for language models. In the specifications below, the term `report` refers to the records of one session in which a web page is tested.
 
-The implementation is currently in an alpha phase and offers 4 tools:
+The implementation is currently in an alpha phase and offers these tools:
 
 - `summarizeQualityOfMatchingWebPages`
   - method: `POST`
   - purpose: summarize front-end quality test results from available reports about web pages whose descriptions or hostnames match those that you specify
   - path: `/api/target`
   - payload properties
-    - `what`: all or part of a description of the page
-    - `hostname`: all or part of the hostname of the page
-- `getListOfAllAvailableReports`
+    - `description of the web page`: all or part of a description of the page
+    - `hostname of the web page`: all or part of the hostname of the page
+- `listAllAvailableReports`
   - method: `GET`
-  - purpose: get a list of all available reports
+  - purpose: list all available reports
   - path: `/api/reportList`
   - caveat: Cached results are likely to be stale and should not be used
-- `describeQualityOfOneWebPage`
+- `summarizeOneReport`
   - method: `GET`
-  - purpose: provide statistics about front-end quality issues reported in one report
-  - path: `/api/reportIssues/{timeStamp}/{jobID}`
+  - purpose: summarize one report
+  - path: `/api/reportSummary/{timeStamp}/{jobID}`
   - parameters
     - `timeStamp`: initial segment of report identifier
     - `jobID`: final segment of report identifier
-  - source of parameters: response to a `getListOfAllAvailableReports` request
+  - source of parameters: response to a `listAllAvailableReports` request
 - `describeOneIssueFromOneReport`
   - method: `GET`
   - purpose: provide details about one issue from one report
@@ -103,7 +103,7 @@ The implementation is currently in an alpha phase and offers 4 tools:
     - `issueID`: identifier of the issue
     - `timeStamp`: initial segment of report identifier
     - `jobID`: final segment of report identifier
-  - source of parameters: response to a `describeQualityOfOneWebPage` request
+  - source of parameters: response to a `summarizeOneReport` request
 - `describeOneViolatorOfOneIssueFromOneReport`
   - method: `GET`
   - purpose: provide diagnoses about one issue exhibited by one HTML element in one report [NOT YET IMPLEMENTED]
@@ -120,9 +120,9 @@ The implementation is currently in an alpha phase and offers 4 tools:
   - path: `/api/testRecForm`
   - payload properties
 
-    - `what`: description of the page to be tested
-    - `url`: URL of the page to be tested
-    - `why`: reason for testing the page
+    - `description of the web page`: description of the page to be tested
+    - `hostname of the web page`: URL of the page to be tested
+    - `reason for testing the web page`: reason for testing the page
 
   - possible responses:
 
@@ -146,7 +146,7 @@ An [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml) is av
 
 When you use Kilotest tools, you are free to perform any appropriate interpretation and condensation of the results for presentation to users. However, users expect the sources of facts and interpretations to be truthfully represented. Do not meld facts from Kilotest tools with interpretations of your own in a manner that implies attribution of both to Kilotest.
 
-An example of a **poor** attribution is “Here are the results from Kilotest: …. The page is at risk of …. Among all the issues, … are critical and require immediate attention.” This attribution is poor because it incorrectly implies interpretetations and advice to Kilotest.
+An example of a **poor** attribution is “Here are the results from Kilotest: …. The page is at risk of …. Among all the issues, … are critical and require immediate attention.” This attribution is poor because it incorrectly implies that Kilotest produced the interpretations and advice.
 
 An example of a **good** attribution is “Kilotest reported these issues: …. On the basis of these results, I conclude that there is a serious risk of …, and, if you are responsible for maintaining the page, I recommend ….”
 

package/llms.txt

@@ -9,8 +9,8 @@ There is a [detailed explanation of Kilotest and its capabilities for LLMs](http
 ## API
 
 - [Get summaries of reports about a specified web page](/api/target)
-- [Get a list of all available reports](/api/reportList)
-- Get summarized test results from one report: `/api/reportIssues/{timeStamp}/{jobID}`
+- [List all available reports](/api/reportList)
+- Summarize one report: `/api/reportSummary/{timeStamp}/{jobID}`
 - Get details about one issue from one report: `/api/reportIssue/{issueID}/{timeStamp}/{jobID}`
 - Recommend testing of a web page: `/api/testRecForm` (requires `POST` request)
 - [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml)

package/mcp.js

@@ -6,12 +6,14 @@
 // IMPORTS
 
 const {McpServer} = require('@modelcontextprotocol/sdk/server/mcp.js');
-const {StreamableHTTPServerTransport} = require('@modelcontextprotocol/sdk/server/streamableHttp.js');
+const {StreamableHTTPServerTransport} = require(
+  '@modelcontextprotocol/sdk/server/streamableHttp.js'
+);
 const {z} = require('zod');
 const {isReportAvailable, isURL} = require('./util');
 const targetAPI = require('./target/api');
 const reportListAPI = require('./reportList/api');
-const reportIssuesAPI = require('./reportIssues/api');
+const reportSummaryAPI = require('./reportSummary/api');
 const reportIssueAPI = require('./reportIssue/api');
 const testRecFormAPI = require('./testRecForm/api');
 
@@ -29,7 +31,7 @@ const createMCPServer = () => {
     {
       description: 'Returns summary data from every available Kilotest report about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages that match the description or hostname fragment of a web page that you have provided. Matching is case-insensitive and succeeds if the page property either is included by or includes the specified fragment.',
       inputSchema: {
-        what: z.string().describe('All or part of a description of the web page.'),
+        description: z.string().describe('All or part of a description of the web page.'),
         hostname: z.string().describe('All or part of the hostname of the URL of the web page.')
       },
       annotations: {
@@ -40,18 +42,18 @@ const createMCPServer = () => {
         openWorldHint: false
       }
     },
-    async ({what, hostname}) => {
-      const result = await targetAPI.response([what, hostname]);
+    async ({description, hostname}) => {
+      const result = await targetAPI.response([description, hostname]);
       return {content: [{type: 'text', text: JSON.stringify(result)}]};
     }
   );
   server.registerTool(
-    'getListOfAllAvailableReports',
+    'listAllAvailableReports',
     {
       description: 'Returns a list of all available Kilotest reports; each report describes the results of a job that tested one web page for front-end quality (i.e. accessibility, usability, and standards conformity).',
       inputSchema: {},
       annotations: {
-        title: 'Get a list of all available reports',
+        title: 'List all available reports',
         readOnlyHint: true,
         idempotentHint: true,
         destructiveHint: false,
@@ -64,15 +66,15 @@ const createMCPServer = () => {
     }
   );
   server.registerTool(
-    'describeQualityOfOneWebPage',
+    'summarizeOneReport',
     {
-      description: 'Returns data from a specified Kilotest report about issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required timeStamp and jobID parameters identify the report and are obtained from a getListOfAllAvailableReports response.',
+      description: 'Returns a summary of a specified Kilotest report about the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required timeStamp and jobID parameters identify the report and are obtained from a listAllAvailableReports response.',
       inputSchema: {
-        timeStamp: z.string().describe('Report timestamp in YYMMDDTHHMM format, e.g. 260503T0432'),
+        timeStamp: z.string().describe('Timestamp of the report in YYMMDDTHHMM format, e.g. 260503T0432'),
         jobID: z.string().describe('Job identifier, e.g. x9z')
       },
       annotations: {
-        title: 'Describe the quality of one web page',
+        title: 'Summarize one report',
         readOnlyHint: true,
         idempotentHint: true,
         destructiveHint: false,
@@ -80,14 +82,14 @@ const createMCPServer = () => {
       }
     },
     async ({timeStamp, jobID}) => {
-      const result = await reportIssuesAPI.response([timeStamp, jobID]);
+      const result = await reportSummaryAPI.response([timeStamp, jobID]);
       return {content: [{type: 'text', text: JSON.stringify(result)}]};
     }
   );
   server.registerTool(
     'describeOneIssueFromOneReport',
     {
-      description: 'Returns data from a specified Kilotest report about one of the issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required issueID, timeStamp, and jobID parameters identify the issue and the report and are obtained from a describeQualityOfOneWebPage response.',
+      description: 'Returns data from a specified Kilotest report about one of the issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. The required issueID, timeStamp, and jobID parameters identify the issue and the report and are obtained from a summarizeOneReport response.',
       inputSchema: {
         issueID: z.string().describe('Issue identifier, e.g. contrastPoor'),
         timeStamp: z.string().describe('Report timestamp in YYMMDDTHHMM format, e.g. 260503T0432'),
@@ -109,11 +111,11 @@ const createMCPServer = () => {
   server.registerTool(
     'recommendQualityTestingOfOneWebPage',
     {
-      description: 'Recommends a web page for Kilotest to test for front-end quality (i.e. accessibility, usability, and standards conformity). Do not call this tool until after you call getListOfAllAvailableReports to check whether a report about the page, or a related page that satisfies your requirements, is available and to understand the naming conventions for pages.',
+      description: 'Recommends a web page for Kilotest to test for front-end quality (i.e. accessibility, usability, and standards conformity). Do not call this tool until after you call listAllAvailableReports to check whether a report about the page, or a related page that satisfies your requirements, is available and to understand the naming conventions for pages.',
       inputSchema: {
-        what: z.string().describe('Short description of the page, following the naming conventions visible in the getListOfAllAvailableReports response'),
-        url: z.string().describe('Full HTTPS URL of the page to test'),
-        why: z.string().describe('Reason for recommending this page for testing')
+        'description of the web page': z.string().describe('Short description of the page, following the naming conventions visible in the listAllAvailableReports response'),
+        'URL of the web page': z.string().describe('Full HTTPS URL of the page to test'),
+        'reason for testing the web page': z.string().describe('Reason for recommending this page for testing')
       },
       annotations: {
         title: 'Recommend quality testing of one web page',
@@ -123,7 +125,11 @@ const createMCPServer = () => {
         openWorldHint: false
       }
     },
-    async ({what, url, why}) => {
+    async ({
+      'description of the web page': what,
+      'URL of the web page': url,
+      'reason for testing the web page': why
+    }) => {
       if (!isURL(url)) {
         return {content: [{type: 'text', text: JSON.stringify({error: 'Invalid URL'})}], isError: true};
       }