@jrpool/kilotest 37.0.0 → 38.0.0

package/reportIssues/util.js

@@ -6,7 +6,7 @@
 // IMPORTS
 
 const {
-  getPageData, getReport, getToolsData, getToolList, isValidReport, objectSort
+  getPageData, getReport, getToolsData, getToolList, isHidden, isValidReport, objectSort
 } = require('../util');
 const issuesClassification = require('testilo/procs/score/tic').issues;
 
@@ -16,7 +16,12 @@ const issuesClassification = require('testilo/procs/score/tic').issues;
 const getIssuesData = async (timeStamp, jobID) => {
   // Get the report.
   const report = await getReport(timeStamp, jobID);
-  // If it is valid:
+  const reportIsHidden = await isHidden(timeStamp, jobID);
+  // If it exists and is hidden:
+  if (reportIsHidden) {
+    return {error: 'Report is not available'}
+  }
+  // Otherwise, if it exists and is valid:
   if (isValidReport(report)) {
     // Initialize the temporary data.
     const temp = {
@@ -110,10 +115,24 @@ const getIssuesData = async (timeStamp, jobID) => {
     // Return the data.
     return final;
   }
-  // Otherwise, i.e. if it is invalid, return this.
-  return 'ERROR: Report missing or invalid.';
+  // Otherwise, i.e. if it is invalid or does not exist, return this.
+  return {error: 'Report missing or invalid.'};
+};
+// Get page and issues data from a report.
+exports.getData = async (timeStamp, jobID) => {
+  const pageData = await getPageData(timeStamp, jobID);
+  const issuesData = await getIssuesData(timeStamp, jobID);
+  const pageError = pageData.error || '';
+  const issuesError = issuesData.error || '';
+  const errors = [pageError, issuesError].filter(Boolean).join('; ');
+  // If the data of either type are missing or invalid:
+  if (errors) {
+    // Return this.
+    return {error: errors};
+  }
+  // Otherwise, return the data.
+  return {
+    pageData,
+    issuesData
+  };
 };
-exports.getData = async (timeStamp, jobID) => ({
-  pageData: await getPageData(timeStamp, jobID),
-  issuesData: await getIssuesData(timeStamp, jobID)
-});

package/reportList/api.js

@@ -22,54 +22,62 @@ const thisHost = process.env.THIS_KILOTEST_HOST;
 
 // Returns a response to an API request for reports.
 exports.response = async () => {
+  // Initialize an array of facts about reports.
   const reportsFacts = [];
   // Get the available logs, with added job names.
   const availableLogs = await getLogs();
-  // For each of them:
+  // For each log:
   for (const availableLog of availableLogs) {
     const {jobName} = availableLog;
     const [timeStamp, jobID] = jobName.split('-');
     const reportSize = await getReportSize(timeStamp, jobID);
-    const {superseded, url, what} = availableLog;
-    reportsFacts.push({
-      identifier: jobName,
-      'creation date and time': getDateTime(timeStamp),
-      'days since the creation date': getAgoDays(timeStamp),
-      'tested web page': {
-        description: what,
-        URL: url
-      },
-      'whether a later report about the same page exists': !! superseded,
-      'URLs for incrementally getting facts from the report': {
-        'for you': `${thisHost}/api/reportIssues/${timeStamp}/${jobID}`,
-        'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
-      },
-      'size of the report in bytes': reportSize,
-      'URL to get the entire report as machine-oriented JSON': `${thisHost}/fullReport.json/${timeStamp}/${jobID}`
-    });
+    // If its report exists:
+    if (reportSize) {
+      const {superseded, url, what} = availableLog;
+      // Add facts about the report to the array.
+      reportsFacts.push({
+        identifier: jobName,
+        'creation date and time': getDateTime(timeStamp),
+        'days since the creation date': getAgoDays(timeStamp),
+        'tested web page': {
+          description: what,
+          URL: url
+        },
+        'whether a later report about the same page exists': !! superseded,
+        'URLs for more details': {
+          'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
+          'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
+        },
+        'size of the report in bytes': reportSize,
+        'URL to get the entire report as machine-oriented JSON': `${thisHost}/fullReport.json/${timeStamp}/${jobID}`
+      });
+    }
+    // Otherwise, i.e. if the report does not exist:
+    else {
+      // Report this.
+      console.error(`Log ${jobName} exists but its report does not.`);
+    }
   }
-  // Get a response.
+  // Create a response body.
   const content = {
     summary: `This document fulfills a request made by a language model to a Kilotest tool. The model asked which reports about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages are available from Kilotest. Each report describes the results of a job that tested one web page. Kilotest, with the help of Testaro, Testilo, and an ensemble of ten rule engines, performs more than a thousand tests on a public web page, using a combination of rule- and machine-learning-based methods. Kilotest exposes API endpoints and web UI URLs to recommend web pages for testing and to obtain information from Kilotest reports. You and your users can learn more about Kilotest and the advantages of testing with an ensemble of rule engines at the deployed instance of Kilotest (${process.env.DEPLOYED_KILOTEST_HOST}).`,
     'tool collection name': 'Kilotest',
-    'tool name': 'getListOfAllAvailableReports',
+    'tool name': 'listAllAvailableReports',
     request: {
-      'type of request': {
-        identifier: 'reportList',
-        description: 'Get a list of all available reports. Each report describes the results of a job that tested a web page for front-end quality (i.e. accessibility, usability, and standards conformity). For each report, the list identifies which page was tested and when and provides URLs to begin incremental retrieval of the test results.'
-      },
+      description: 'List all available reports. For each report, the list should identify which page was tested and when and provide URLs to begin incremental retrieval of the test results from the report.',
       method: 'GET',
       URLs: {
-        'URL of your request': `${thisHost}/api/reportList`,
-        'equivalent URL for humans': `${thisHost}/targets.html`
+        'for you': `${thisHost}/api/reportList`,
+        'for humans': `${thisHost}/targets.html`
       },
       'closest ancestor request': null
     },
     'response metadata': {
       identifier: `${getNowStamp()}-${getRandomString(3)}`,
-      'date and time': new Date().toISOString(),
+      'date and time': new Date().toISOString()
     },
-    'available reports': reportsFacts
+    'response content': reportsFacts
   };
+  // Return it.
   return content;
 };

package/reportSummary/api.js

@@ -0,0 +1,181 @@
+/*
+  api.js
+  Responds to the reportSummary API request.
+*/
+
+// IMPORTS
+
+const {
+  getNowStamp,
+  getRandomString,
+  getReport,
+  isHidden,
+  objectSort,
+  ruleEngines
+} = require('../util');
+const issuesClassification = require('testilo/procs/score/tic').issues;
+
+// CONSTANTS
+
+const thisHost = process.env.THIS_KILOTEST_HOST;
+
+// FUNCTIONS
+
+// Returns facts about a report.
+const getReportFacts = async (report) => {
+  // Initialize the facts.
+  const facts = {
+    ruleEngineIDs: new Set(),
+    reporterIDs: new Set(),
+    issueIDs: new Set(),
+    violators: new Set()
+  };
+  // For each act in the report:
+  report.acts.forEach(act => {
+    // If it is a test act:
+    if (act.type === 'test') {
+      const {result, which} = act;
+      // Ensure its rule engine is in the facts.
+      facts.ruleEngineIDs.add(which);
+      const instances = result?.standardResult?.instances ?? [];
+      // For each of its standard instances:
+      instances.forEach(instance => {
+        const {catalogIndex, issueID} = instance;
+        // If the instance identifies a non-ignorable issue:
+        if (issueID && issueID !== 'ignorable') {
+          const issueClassification = issuesClassification[issueID];
+          // If the issue is classified and has a valid weight:
+          if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
+            // Ensure the issue ID is in the facts.
+            facts.issueIDs.add(issueID);
+            // Ensure its rule engine is among the reporters in the facts.
+            facts.reporterIDs.add(which);
+            // If the instance has a catalog index:
+            if (catalogIndex) {
+              // Ensure the violator is in the facts.
+              facts.violators.add(catalogIndex);
+            }
+          }
+        }
+      });
+    }
+  });
+  // Return the facts.
+  return facts;
+};
+// Returns facts about a rule engine.
+const getRuleEngineFacts = ruleEngineID => {
+  const ruleEngineData = ruleEngines[ruleEngineID];
+  return {
+    identifier: ruleEngineID,
+    name: ruleEngineData[0],
+    sponsor: ruleEngineData[1]
+  };
+};
+// Returns facts about rule engines that were prevented from testing the page.
+const getPreventionFacts = report => {
+  return objectSort(Object.entries(report.jobData.preventions).map(([ruleEngineID, reason]) => ({
+    'name': getRuleEngineFacts(ruleEngineID).name,
+    'reason for failure': reason
+  })), 'name', 'alpha');
+};
+// Returns rule engine IDs sorted by name.
+const getSortedRuleEngineIDs = ruleEngineIDSet => {
+  return Array.from(ruleEngineIDSet).sort((a, b) => {
+    const aData = ruleEngines[a][0];
+    const bData = ruleEngines[b][0];
+    return aData.localeCompare(bData, 'en', {sensitivity: 'base'});
+  });
+};
+// Returns issue IDs sorted by priority and summary.
+const getSortedIssueIDs = issueIDSet => {
+  return Array.from(issueIDSet).sort((a, b) => {
+    const aData = issuesClassification[a];
+    const bData = issuesClassification[b];
+    if (aData.weight !== bData.weight) {
+      return bData.weight - aData.weight;
+    }
+    return aData.summary.localeCompare(bData.summary, 'en', {sensitivity: 'base'});
+  });
+};
+// Get facts about an issue.
+const getIssueFacts = (issueID, timeStamp, jobID) => {
+  const issueData = issuesClassification[issueID];
+  const {summary, weight, why} = issueData;
+  return {
+    identifier: issueID,
+    summary,
+    'impact on a user': why,
+    'priority': ['lowest', 'low', 'high', 'highest'][weight - 1],
+    'URLs for more details': {
+      'for you': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
+      'for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
+    }
+  };
+};
+// Returns a response to an API request for a summary of one report.
+exports.response = async (args) => {
+  const [timeStamp, jobID] = args;
+  const reportIsHidden = await isHidden(timeStamp, jobID);
+  // If the report exists and is hidden:
+  if (reportIsHidden) {
+    // Return this.
+    return {
+      status: 'error',
+      message: 'Report not available',
+    };
+  }
+  // Otherwise, i.e. if the report is not hidden, get it.
+  const report = await getReport(timeStamp, jobID);
+  const {error} = report;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
+  // Otherwise, i.e. if it succeeded, get facts about it.
+  const reportFacts = await getReportFacts(report);
+  // Create a response body.
+  const content = {
+    summary: 'This document fulfills a request made by a language model to a Kilotest tool. The model asked for a summary of one Kilotest report. The model had previously used the listAllAvailableReports tool and had acquired from that tool basic facts about Kilotest, the ensemble testing that Kilotest performs, and the reports available from Kilotest. This summary does not repeat those facts.',
+    'tool collection name': 'Kilotest',
+    'tool name': 'summarizeOneReport',
+    request: {
+      description: 'Summarize one report. The summary should briefly describe the testing job and the results, including the rule engines that tested the web page and the issues that were revealed by the reported rule violations, and provide URLs for getting details about issues. The timeStamp and jobID parameters identify the report and were obtained from the response to a listAllAvailableReports operation.',
+      method: 'GET',
+      URLs: {
+        'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
+        'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
+      },
+      'closest ancestor request': {
+        'tool name': 'listAllAvailableReports',
+        description: 'List all available reports.',
+        URLs: {
+          'for you': `${thisHost}/api/reportList`,
+          'for humans': `${thisHost}/targets.html`
+        }
+      }
+    },
+    'response metadata': {
+      identifier: `${getNowStamp()}-${getRandomString(3)}`,
+      'date and time': new Date().toISOString()
+    },
+    'response content': {
+      'rule engines that tried to test the page': getSortedRuleEngineIDs(reportFacts.ruleEngineIDs)
+      .map(id => getRuleEngineFacts(id)),
+      'rule engines that could not test the page': getPreventionFacts(report),
+      'names of rule engines that reported rule violations': Array
+      .from(reportFacts.reporterIDs)
+      .map(id => getRuleEngineFacts(id).name)
+      .sort((a, b) => a.localeCompare(b, 'en', {sensitivity: 'base'})),
+      'number of elements reported as violators': reportFacts.violators.size,
+      'issues revealed': getSortedIssueIDs(reportFacts.issueIDs)
+      .map(id => getIssueFacts(id, timeStamp, jobID))
+    }
+  };
+  // Return it.
+  return content;
+};

package/reportUnhideForm/index.js

@@ -22,7 +22,15 @@ exports.answer = async (_, search) => {
     if (authCode === process.env.AUTH_CODE) {
       // Get the log of the report.
       const log = await getLog(... jobName.split('-'));
-      // Reverse the hiddenness property of the log.
+      // If this failed:
+      if (log.error) {
+        // Return why.
+        return {
+          status: 'error',
+          message: log.error
+        }
+      }
+      // Otherwise, i.e. if it succeeded, reverse the hiddenness property of the log.
       log.hidden = false;
       // Save the updated log.
       await fs.writeFile(path.join(logsPath, `${jobName}.json`), getJSON(log));
@@ -37,13 +45,23 @@ exports.answer = async (_, search) => {
     }
   }
   const reportFileNames = await fs.readdir(reportsPath);
+  // Initialize an array of report specifications.
   const reportSpecs = [];
   // For each report:
   for (const reportFileName of reportFileNames) {
     const [timeStamp, jobID] = reportFileName.slice(0, -5).split('-');
     // Get its log.
     const log = await getLog(timeStamp, jobID);
-    const {what, hidden} = log;
+    const {error, what, hidden} = log;
+    // If this failed:
+    if (error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: error
+      }
+    }
+    // Otherwise, i.e. if it succeeded, add the report to the array.
     reportSpecs.push({
       what,
       timeStamp,

package/reportsExpungeForm/index.js

@@ -44,7 +44,15 @@ exports.answer = async (_, search) => {
     const [timeStamp, jobID] = reportName.slice(0, -5).split('-');
     // Get a summary of it.
     const reportSummary = await getReportData(timeStamp, jobID);
-    const {issueCount, preventedToolCount, url} = reportSummary;
+    const {error, issueCount, preventedToolCount, url} = reportSummary;
+    // If this failed:
+    if (error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: error
+      }
+    }
     reportSpecs.push({
       timeStamp,
       jobID,

package/reportsPruneForm/index.js

@@ -38,13 +38,23 @@ exports.answer = async (_, search) => {
     }
   }
   const reportNames = await fs.readdir(reportsPath);
+  // Initialize an array of report summaries.
   const reportSpecs = [];
   // For each report:
   for (const reportName of reportNames) {
     const [timeStamp, jobID] = reportName.slice(0, -5).split('-');
     // Get a summary of it.
     const reportSummary = await getReportData(timeStamp, jobID);
-    const {issueCount, preventedToolCount, url} = reportSummary;
+    const {error, issueCount, preventedToolCount, url} = reportSummary;
+    // If this failed:
+    if (error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: error
+      }
+    }
+    // Otherwise, i.e. if it succeeded, add the summary to the array.
     reportSpecs.push({
       timeStamp,
       jobID,

package/reportsRewindForm/index.js

@@ -44,7 +44,15 @@ exports.answer = async (_, search) => {
     const [timeStamp, jobID] = reportName.slice(0, -5).split('-');
     // Get a summary of it.
     const reportSummary = await getReportData(timeStamp, jobID);
-    const {issueCount, preventedToolCount, url} = reportSummary;
+    const {error, issueCount, preventedToolCount, url} = reportSummary;
+    // If this failed:
+    if (error) {
+      // Return why.
+      return {
+        status: 'error',
+        message: error
+      }
+    }
     reportSpecs.push({
       timeStamp,
       jobID,

package/researchAgent.js

@@ -80,11 +80,10 @@ const getRequestOptions = (path, method = 'GET') => ({
   }
 });
 // Submits a request, returns the response content, and increments the results.
-const submitRequest = async (results, path, method, body = null) => new Promise(resolve => {
+const submitRequest = async (path, method, body = null) => new Promise(resolve => {
   console.log(`Making ${scheme} ${method} request on port ${port} to ${host}${path}`);
   client.request(getRequestOptions(path, method), async response => {
     const responseContent = await getContent(response);
-    results.push(responseContent.error ? 'bad' : 'good');
     resolve(responseContent);
   })
   .on('error', error => {
@@ -104,26 +103,25 @@ const requestService = async () => {
   let jobID;
   let description;
   let url;
-  const results = [];
-  console.log('======================\nRequest 1: Summarize nonexistent reports');
+  console.log('======================\nRequest: Summarize nonexistent reports');
   method = 'POST';
   path = '/api/target';
-  content = await submitRequest(results, path, method, {
-    what: 'oesntuhaesouht',
+  content = await submitRequest(path, method, {
+    description: 'oesntuhaesouht',
     hostname: 'osentuhaoesuht.aoesntuh'
   });
   if (content.error) {
     return;
   }
-  console.log('======================\nRequest 2: List all available reports');
+  console.log('======================\nRequest: List all available reports');
   method = 'GET';
   path = '/api/reportList';
-  content = await submitRequest(results, path, method);
-  reports = content?.['available reports'] ?? [];
+  content = await submitRequest(path, method);
+  reports = content?.['response content'] ?? [];
   if (content.error || ! Array.isArray(reports) || ! reports.length) {
     return;
   }
-  console.log('======================\nRequest 3: Summarize matching reports');
+  console.log('======================\nRequest: Summarize matching reports');
   // Choose one available report at random.
   report = reports[Math.floor(Math.random() * reports.length)];
   ({description, URL: url} = report?.['tested web page'] ?? ['', '']);
@@ -132,61 +130,72 @@ const requestService = async () => {
   }
   method = 'POST';
   path = '/api/target';
-  content = await submitRequest(results, path, method, {
-    what: description,
+  content = await submitRequest(path, method, {
+    description,
     hostname: new URL(url).hostname ?? ''
   });
   if (content.error) {
     return;
   }
-  console.log('======================\nRequest 4: Summarize the issues in a report');
+  console.log('======================\nRequest: Summarize one nonexistent report');
+  [timeStamp, jobID] = ['111111T1111', 'abc'];
+  method = 'GET';
+  path = `/api/reportSummary/${timeStamp}/${jobID}`;
+  content = await submitRequest(path, method);
+  if (! content.message) {
+    return;
+  }
+  console.log('======================\nRequest: Summarize one report');
   [timeStamp, jobID] = report.identifier?.split('-') ?? ['', ''];
   if (! (timeStamp && jobID)) {
     return;
   }
   method = 'GET';
-  path = `/api/reportIssues/${timeStamp}/${jobID}`;
-  content = await submitRequest(results, path, method);
-  if (content.error) {
+  path = `/api/reportSummary/${timeStamp}/${jobID}`;
+  content = await submitRequest(path, method);
+  if (content.message || ! content.summary) {
     return;
   }
-  console.log('======================\nRequest 5: Describe one issue from one report');
-  if (content['number of HTML elements reported as exhibiting issues'] === 0) {
+  console.log('======================\nRequest: Describe one issue from one report');
+  if (content['response content']['number of elements reported as violators'] === 0) {
     console.log('reportIssue request cannot be submitted, because no issues were reported');
   }
   else {
     // Get the issue IDs.
     const issueIDs = Object
-    .values(content['issues reported'])
-    .map(levelIssues => levelIssues.map(issue => issue.identifier))
-    .flat();
+    .values(content['response content']['issues revealed'])
+    .map(issue => issue.identifier);
     // Choose one at random.
     const issueID = issueIDs[Math.floor(Math.random() * issueIDs.length)];
     method = 'GET';
     path = `/api/reportIssue/${issueID}/${timeStamp}/${jobID}`;
-    content = await submitRequest(results, path, method);
-    if (content.error) {
+    content = await submitRequest(path, method);
+    if (content.message) {
       return;
     }
   }
-  console.log('======================\nRequest 6: Make a permitted test recommendation');
+  console.log('======================\nRequest: Make a permitted test recommendation');
   method = 'POST';
   path = '/api/testRecForm';
-  content = await submitRequest(results, path, method, {
+  content = await submitRequest(path, method, {
     'description of the web page': 'aoseeou',
     'URL of the web page': 'https://oaaestuh.osneth',
     'reason for testing the web page': 'Just testing'
   });
-  if (content.error) {
+  if (content.message) {
     return;
   }
-  console.log('======================\nRequest 7: Make an illicit test recommendation');
-  await submitRequest(results, path, method, {
+  console.log('======================\nRequest: Make an illicit test recommendation');
+  content = await submitRequest(path, method, {
     'description of the web page': description,
     'URL of the web page': url,
     'reason for testing the web page': 'Just testing'
   });
-  console.log(`======================\nResults: ${results}\n`);
+  if (! content.message) {
+    return;
+  }
+  console.log('======================\nRequest: Results');
+  console.log('All tests succeeded');
 };
 
 // EXECUTION

package/retestRec/index.js

@@ -12,8 +12,17 @@ const {getLog, processRec} = require('../util');
 // Records a retest recommendation and returns an acknowledgement page.
 exports.answer = async (pageArgs, why) => {
   const [timeStamp, jobID] = pageArgs.split('/');
+  // Get the log of the specified report.
   const log = await getLog(timeStamp, jobID);
-  const {url, what} = log;
-  // Process the recommendation.
+  const {error, url, what} = log;
+  // If this failed:
+  if (error) {
+    // Return why.
+    return {
+      status: 'error',
+      message: error
+    };
+  }
+  // Otherwise, i.e. if it succeeded, process the recommendation.
   return await processRec('retest', __dirname, what, url, why);
 };

package/retestRecForm/index.js

@@ -15,6 +15,11 @@ const path = require('path');
 exports.answer = async pageArgs => {
   const [timeStamp, jobID] = pageArgs.split('/');
   const log = await getLog(timeStamp, jobID);
+  // If this failed:
+  if (log.error) {
+    // Make the form report the failure.
+    log.what = 'The specified page is not available for retesting';
+  }
   const query = {
     target: log.what,
     timeStamp,