@jrpool/kilotest 38.1.0 → 39.0.1

package/api/reportList.js

@@ -1,83 +0,0 @@
-/*
-  api.js
-  Responds to the reportList API request.
-*/
-
-// IMPORTS
-
-const {
-  getAgoDays,
-  getDateTime,
-  getLogs,
-  getNowStamp,
-  getRandomString,
-  getReportSize
-} = require('../util');
-
-// CONSTANTS
-
-const thisHost = process.env.THIS_KILOTEST_HOST;
-
-// FUNCTIONS
-
-// Returns a response to an API request for reports.
-exports.response = async () => {
-  // Initialize an array of facts about reports.
-  const reportsFacts = [];
-  // Get the available logs, with added job names.
-  const availableLogs = await getLogs();
-  // For each log:
-  for (const availableLog of availableLogs) {
-    const {jobName} = availableLog;
-    const [timeStamp, jobID] = jobName.split('-');
-    const reportSize = await getReportSize(timeStamp, jobID);
-    // If its report exists:
-    if (reportSize) {
-      const {superseded, url, what} = availableLog;
-      // Add facts about the report to the array.
-      reportsFacts.push({
-        identifier: jobName,
-        'creation date and time': getDateTime(timeStamp),
-        'days since the creation date': getAgoDays(timeStamp),
-        'tested web page': {
-          description: what,
-          URL: url
-        },
-        'whether a later report about the same page exists': !! superseded,
-        'URLs for more details': {
-          'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
-          'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
-        },
-        'size of the report in bytes': reportSize,
-        'URL to get the entire report as machine-oriented JSON': `${thisHost}/fullReport.json/${timeStamp}/${jobID}`
-      });
-    }
-    // Otherwise, i.e. if the report does not exist:
-    else {
-      // Report this.
-      console.error(`Log ${jobName} exists but its report does not.`);
-    }
-  }
-  // Create a response body.
-  const content = {
-    summary: `This document fulfills a request made by a language model to a Kilotest tool. The model asked which reports about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages are available from Kilotest. Each report describes the results of a job that tested one web page. Kilotest, with the help of Testaro, Testilo, and an ensemble of ten rule engines, performs more than a thousand tests on a public web page, using a combination of rule- and machine-learning-based methods. Kilotest exposes API endpoints and web UI URLs to recommend web pages for testing and to obtain information from Kilotest reports. You and your users can learn more about Kilotest and the advantages of testing with an ensemble of rule engines at the deployed instance of Kilotest (${process.env.DEPLOYED_KILOTEST_HOST}).`,
-    'tool collection name': 'Kilotest',
-    'tool name': 'listAllAvailableReports',
-    request: {
-      description: 'List all available reports. For each report, the list should identify which page was tested and when and provide URLs to begin incremental retrieval of the test results from the report.',
-      method: 'GET',
-      URLs: {
-        'for you': `${thisHost}/api/reportList`,
-        'for humans': `${thisHost}/targets.html`
-      },
-      'closest ancestor request': null
-    },
-    'response metadata': {
-      identifier: `${getNowStamp()}-${getRandomString(3)}`,
-      'date and time': new Date().toISOString()
-    },
-    'response content': reportsFacts
-  };
-  // Return it.
-  return content;
-};

package/api/reportSummary.js

@@ -1,181 +0,0 @@
-/*
-  api.js
-  Responds to the reportSummary API request.
-*/
-
-// IMPORTS
-
-const {
-  getNowStamp,
-  getRandomString,
-  getReport,
-  isHidden,
-  objectSort,
-  ruleEngines
-} = require('../util');
-const issuesClassification = require('testilo/procs/score/tic').issues;
-
-// CONSTANTS
-
-const thisHost = process.env.THIS_KILOTEST_HOST;
-
-// FUNCTIONS
-
-// Returns facts about a report.
-const getReportFacts = async (report) => {
-  // Initialize the facts.
-  const facts = {
-    ruleEngineIDs: new Set(),
-    reporterIDs: new Set(),
-    issueIDs: new Set(),
-    violators: new Set()
-  };
-  // For each act in the report:
-  report.acts.forEach(act => {
-    // If it is a test act:
-    if (act.type === 'test') {
-      const {result, which} = act;
-      // Ensure its rule engine is in the facts.
-      facts.ruleEngineIDs.add(which);
-      const instances = result?.standardResult?.instances ?? [];
-      // For each of its standard instances:
-      instances.forEach(instance => {
-        const {catalogIndex, issueID} = instance;
-        // If the instance identifies a non-ignorable issue:
-        if (issueID && issueID !== 'ignorable') {
-          const issueClassification = issuesClassification[issueID];
-          // If the issue is classified and has a valid weight:
-          if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
-            // Ensure the issue ID is in the facts.
-            facts.issueIDs.add(issueID);
-            // Ensure its rule engine is among the reporters in the facts.
-            facts.reporterIDs.add(which);
-            // If the instance has a catalog index:
-            if (catalogIndex) {
-              // Ensure the violator is in the facts.
-              facts.violators.add(catalogIndex);
-            }
-          }
-        }
-      });
-    }
-  });
-  // Return the facts.
-  return facts;
-};
-// Returns facts about a rule engine.
-const getRuleEngineFacts = ruleEngineID => {
-  const ruleEngineData = ruleEngines[ruleEngineID];
-  return {
-    identifier: ruleEngineID,
-    name: ruleEngineData[0],
-    sponsor: ruleEngineData[1]
-  };
-};
-// Returns facts about rule engines that were prevented from testing the page.
-const getPreventionFacts = report => {
-  return objectSort(Object.entries(report.jobData.preventions).map(([ruleEngineID, reason]) => ({
-    'name': getRuleEngineFacts(ruleEngineID).name,
-    'reason for failure': reason
-  })), 'name', 'alpha');
-};
-// Returns rule engine IDs sorted by name.
-const getSortedRuleEngineIDs = ruleEngineIDSet => {
-  return Array.from(ruleEngineIDSet).sort((a, b) => {
-    const aData = ruleEngines[a][0];
-    const bData = ruleEngines[b][0];
-    return aData.localeCompare(bData, 'en', {sensitivity: 'base'});
-  });
-};
-// Returns issue IDs sorted by priority and summary.
-const getSortedIssueIDs = issueIDSet => {
-  return Array.from(issueIDSet).sort((a, b) => {
-    const aData = issuesClassification[a];
-    const bData = issuesClassification[b];
-    if (aData.weight !== bData.weight) {
-      return bData.weight - aData.weight;
-    }
-    return aData.summary.localeCompare(bData.summary, 'en', {sensitivity: 'base'});
-  });
-};
-// Get facts about an issue.
-const getIssueFacts = (issueID, timeStamp, jobID) => {
-  const issueData = issuesClassification[issueID];
-  const {summary, weight, why} = issueData;
-  return {
-    identifier: issueID,
-    summary,
-    'impact on a user': why,
-    'priority': ['lowest', 'low', 'high', 'highest'][weight - 1],
-    'URLs for more details': {
-      'for you': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
-      'for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
-    }
-  };
-};
-// Returns a response to an API request for a summary of one report.
-exports.response = async (args) => {
-  const [timeStamp, jobID] = args;
-  const reportIsHidden = await isHidden(timeStamp, jobID);
-  // If the report exists and is hidden:
-  if (reportIsHidden) {
-    // Return this.
-    return {
-      status: 'error',
-      message: 'Report not available',
-    };
-  }
-  // Otherwise, i.e. if the report is not hidden, get it.
-  const report = await getReport(timeStamp, jobID);
-  const {error} = report;
-  // If this failed:
-  if (error) {
-    // Return why.
-    return {
-      status: 'error',
-      message: error
-    };
-  }
-  // Otherwise, i.e. if it succeeded, get facts about it.
-  const reportFacts = await getReportFacts(report);
-  // Create a response body.
-  const content = {
-    summary: 'This document fulfills a request made by a language model to a Kilotest tool. The model asked for a summary of one Kilotest report. The model had previously used the listAllAvailableReports tool and had acquired from that tool basic facts about Kilotest, the ensemble testing that Kilotest performs, and the reports available from Kilotest. This summary does not repeat those facts.',
-    'tool collection name': 'Kilotest',
-    'tool name': 'summarizeOneReport',
-    request: {
-      description: 'Summarize one report. The summary should briefly describe the testing job and the results, including the rule engines that tested the web page and the issues that were revealed by the reported rule violations, and provide URLs for getting details about issues. The timeStamp and jobID parameters identify the report and were obtained from the response to a listAllAvailableReports operation.',
-      method: 'GET',
-      URLs: {
-        'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
-        'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
-      },
-      'closest ancestor request': {
-        'tool name': 'listAllAvailableReports',
-        description: 'List all available reports.',
-        URLs: {
-          'for you': `${thisHost}/api/reportList`,
-          'for humans': `${thisHost}/targets.html`
-        }
-      }
-    },
-    'response metadata': {
-      identifier: `${getNowStamp()}-${getRandomString(3)}`,
-      'date and time': new Date().toISOString()
-    },
-    'response content': {
-      'rule engines that tried to test the page': getSortedRuleEngineIDs(reportFacts.ruleEngineIDs)
-      .map(id => getRuleEngineFacts(id)),
-      'rule engines that could not test the page': getPreventionFacts(report),
-      'names of rule engines that reported rule violations': Array
-      .from(reportFacts.reporterIDs)
-      .map(id => getRuleEngineFacts(id).name)
-      .sort((a, b) => a.localeCompare(b, 'en', {sensitivity: 'base'})),
-      'number of elements reported as violators': reportFacts.violators.size,
-      'issues revealed': getSortedIssueIDs(reportFacts.issueIDs)
-      .map(id => getIssueFacts(id, timeStamp, jobID))
-    }
-  };
-  // Return it.
-  return content;
-};

package/reportIssue/api.js

@@ -1,183 +0,0 @@
-/*
-  api.js
-  Responds to the report-issue request.
-*/
-
-// IMPORTS
-
-const {getData} = require('../reportIssues/util');
-const {
-  getDateTime,
-  getNowStamp,
-  getRandomString,
-  getReport,
-  getToolsFacts,
-  isHidden,
-  tools
-} = require('../util');
-
-// FUNCTIONS
-
-// Gets facts about an issue.
-const getIssueFacts = async (timeStamp, jobID, issue) => {
-  const {issueID, reporterCount, reporters, summary, violatorCount, wcag, why} = issue;
-  const wcagType = wcag.length === 3 ? 'guideline' : 'success criterion';
-  // Get the report.
-  const report = await getReport(timeStamp, jobID);
-  const {acts, catalog, error} = report;
-  // If this failed:
-  if (error) {
-    // Return why.
-    return {
-      status: 'error',
-      message: error
-    };
-  }
-  const violatorIndexSet = new Set();
-  // For each act of the report:
-  acts.forEach(act => {
-    // If it is a test act:
-    if (act.type === 'test') {
-      const {result} = act;
-      const instances = result?.standardResult?.instances ?? [];
-      // For each standard instance of the act:
-      instances.forEach(instance => {
-        const {catalogIndex} = instance;
-        // If its issue is the issue to be described and the instance has a catalog index:
-        if (instance.issueID === issueID && catalogIndex) {
-          // Ensure that the violator is in the temporary data.
-          violatorIndexSet.add(catalogIndex);
-        }
-      });
-    }
-  });
-  return {
-    identifier: issueID,
-    summary,
-    'related WCAG 2.2 standard': {
-      layer: wcagType,
-      'numeric identifier': wcag
-    },
-    'impact on a user': why,
-    'rule engines reporting the issue': {
-      'number': reporterCount,
-      'names': reporters.map(tool => tool.toolName)
-    },
-    'number of HTML elements reported as exhibiting the issue': violatorCount,
-    'HTML elements reported as exhibiting the issue': Array
-    .from(violatorIndexSet)
-    .filter(index => catalog[index])
-    .map(index => ({
-      identifier: String(index),
-      'tag name': catalog[index].tagName,
-      'id attribute': catalog[index].id,
-      'start tag': catalog[index].startTag,
-      'inner text': catalog[index].text,
-      'inner text usable as a text fragment for linking': catalog[index].textLinkable,
-      'x, y, width, height in pixels': catalog[index].boxID.split(':'),
-      'XPath': catalog[index].pathID
-    }))
-  };
-};
-// Returns a response to a report-issue request.
-exports.response = async args => {
-  const [issueID, timeStamp, jobID] = args;
-  const reportIsHidden = await isHidden(timeStamp, jobID);
-  // If the report is not available:
-  if (reportIsHidden) {
-    // Return this.
-    return {
-      status: 'error',
-      message: 'Report not available'
-    };
-  }
-  // Otherwise, i.e. if the report is available, get data on the report and its issues.
-  const data = await getData(timeStamp, jobID);
-  const {error, issuesData, pageData} = data;
-  // If this failed:
-  if (error) {
-    // Return why.
-    return {
-      status: 'error',
-      message: error
-    };
-  }
-  const {what, url, daysAgo} = pageData;
-  const {issueCount, issues, preventions, reporterCount, reporters, violatorCount} = issuesData;
-  let issue;
-  // Otherwise, i.e. if it succeeded, get the level of and the data on the issue.
-  const issueLevel = [4, 3, 2, 1].find(level => {
-    issue = issues[level].find(issue => issue.issueID === issueID);
-    return issue;
-  });
-  // If the issue was not one of those in the report:
-  if (! issue) {
-    // Return this.
-    return {
-      status: 'error',
-      message: 'Issue not found in the report'
-    };
-  }
-  const preventedTools = Object.entries(preventions).map(prevention => ({
-    name: tools[prevention[0]][0],
-    'reason for failure': prevention[1]
-  }));
-  const thisHost = process.env.THIS_KILOTEST_HOST;
-  // Get a response.
-  const content = {
-    summary: `This document fulfills a request made by a language model to a Kilotest tool. The model requested data, drawn from a Kilotest report, about one of the issues for the front-end quality (i.e. accessibility, usability, and standards conformity) of a web page. Kilotest, with the help of Testaro, Testilo, and an ensemble of ten rule engines, performs tests on web pages, using a combination of rule- and machine-learning-based methods, and produces reports. Kilotest exposes several API endpoints to recommend web pages for testing and to obtain information from Kilotest reports. To learn more about Kilotest and the advantages of testing with an ensemble of rule engines, visit the deployed instance of Kilotest (${process.env.DEPLOYED_KILOTEST_HOST}), which contains an introduction on its home page and a tutorial.`,
-    'tool collection name': 'Kilotest',
-    'tool name': 'describeOneIssueFromOneReport',
-    request: {
-      'type of request': {
-        identifier: 'reportIssue',
-        description: 'Describe one issue from one report.'
-      },
-      method: 'GET',
-      URLs: {
-        'for you': `${thisHost}/api/reportIssue/${issueID}/${timeStamp}/${jobID}`,
-        'for humans': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
-      },
-      'closest ancestor request': {
-        identifier: 'summarizeOneReport',
-        description: 'Summarize one report.',
-        URLs: {
-          'for you': `${thisHost}/api/reportSummary/${timeStamp}/${jobID}`,
-          'for humans': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
-        }
-      }
-    },
-    'response metadata': {
-      identifier: `${getNowStamp()}-${getRandomString(3)}`,
-      'date and time': new Date().toISOString()
-    },
-    report: {
-      identifier: `${timeStamp}-${jobID}`,
-      'creation date': getDateTime(timeStamp),
-      'days since the creation date': daysAgo
-    },
-    'tested web page': {
-      description: what,
-      URL: url
-    },
-    'rule engines that tried to test the page': getToolsFacts(Object.keys(tools)),
-    'rule engines that were unable to test the page': preventedTools,
-    'rule engines that reported issues': {
-      number: reporterCount,
-      names: reporters.map(tool => tool.toolName)
-    },
-    'number of issues reported': {
-      total: issueCount,
-      'by priority': {
-        'highest priority': issues[4].length,
-        'high priority': issues[3].length,
-        'low priority': issues[2].length,
-        'lowest priority': issues[1].length
-      }
-    },
-    'number of HTML elements reported as exhibiting issues': violatorCount,
-    'level of the issue': issueLevel,
-    'facts about the issue': await getIssueFacts(timeStamp, jobID, issue)
-  };
-  return content;
-};

package/reportIssues/util.js

@@ -1,138 +0,0 @@
-/*
-  util.js
-  Shared utilities.
-*/
-
-// IMPORTS
-
-const {
-  getPageData, getReport, getToolsData, getToolList, isHidden, isValidReport, objectSort
-} = require('../util');
-const issuesClassification = require('testilo/procs/score/tic').issues;
-
-// FUNCTIONS
-
-// Returns data on the issues reported by a report.
-const getIssuesData = async (timeStamp, jobID) => {
-  // Get the report.
-  const report = await getReport(timeStamp, jobID);
-  const reportIsHidden = await isHidden(timeStamp, jobID);
-  // If it exists and is hidden:
-  if (reportIsHidden) {
-    return {error: 'Report is not available'}
-  }
-  // Otherwise, if it exists and is valid:
-  if (isValidReport(report)) {
-    // Initialize the temporary data.
-    const temp = {
-      issues: {},
-      reporters: new Set(),
-      violators: new Set()
-    };
-    // Initialize the final data.
-    const final = {
-      reporters: [],
-      reporterList: '',
-      reporterCount: 0,
-      violatorCount: 0,
-      preventions: report.jobData.preventions,
-      issues: {
-        4: [],
-        3: [],
-        2: [],
-        1: []
-      },
-      issueCount: 0
-    };
-    // For each act in the report:
-    report.acts.forEach(act => {
-      // If it is a test act:
-      if (act.type === 'test') {
-        const {result, which} = act;
-        const instances = result?.standardResult?.instances ?? [];
-        // For each of its standard instances:
-        instances.forEach(instance => {
-          const {catalogIndex, issueID} = instance;
-          // If the instance identifies its rule as belonging to a non-ignorable issue:
-          if (issueID && issueID !== 'ignorable') {
-            const issueClassification = issuesClassification[issueID];
-            // If the issue has a current weighted classification:
-            if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
-              const {summary, wcag, weight, why} = issueClassification;
-              // Initialize the temporary data on the issue if necessary.
-              temp.issues[issueID] ??= {
-                issueID,
-                summary,
-                wcag,
-                why,
-                weight,
-                reporters: new Set(),
-                reporterList: '',
-                violators: new Set()
-              };
-              // Ensure the tool is in the temporary data.
-              temp.issues[issueID].reporters.add(which);
-              temp.reporters.add(which);
-              // If the instance has a catalog index:
-              if (catalogIndex) {
-                // Ensure the violator is in the temporary data.
-                temp.issues[issueID].violators.add(catalogIndex);
-                temp.violators.add(catalogIndex);
-              }
-            }
-          }
-        });
-      }
-    });
-    // Finish populating the final data.
-    final.reporters = getToolsData(temp.reporters);
-    final.reporterList = getToolList(temp.reporters);
-    final.reporterCount = final.reporters.length;
-    final.violatorCount = temp.violators.size;
-    Object.values(temp.issues).forEach(issue => {
-      const {issueID, summary, wcag, why, weight} = issue;
-      const finalIssue = {
-        issueID,
-        summary,
-        wcag,
-        why,
-        weight
-      };
-      finalIssue.reporters = getToolsData(issue.reporters);
-      finalIssue.reporterList = getToolList(issue.reporters);
-      finalIssue.reporterCount = finalIssue.reporters.length;
-      finalIssue.violatorCount = issue.violators.size;
-      final.issues[issue.weight].push(finalIssue);
-    });
-    final.issueCount = Object.keys(temp.issues).length;
-    // For each weight:
-    [4, 3, 2, 1].forEach(weight => {
-      // Sort its issues in the final data alphabetically by reporter names.
-      objectSort(final.issues[weight], 'reporterList', 'alpha');
-      // Sort the issues again in descending reporter-count order, making this the primary order.
-      objectSort(final.issues[weight], 'reporterCount', 'numericDown');
-    });
-    // Return the data.
-    return final;
-  }
-  // Otherwise, i.e. if it is invalid or does not exist, return this.
-  return {error: 'Report missing or invalid.'};
-};
-// Get page and issues data from a report.
-exports.getData = async (timeStamp, jobID) => {
-  const pageData = await getPageData(timeStamp, jobID);
-  const issuesData = await getIssuesData(timeStamp, jobID);
-  const pageError = pageData.error || '';
-  const issuesError = issuesData.error || '';
-  const errors = [pageError, issuesError].filter(Boolean).join('; ');
-  // If the data of either type are missing or invalid:
-  if (errors) {
-    // Return this.
-    return {error: errors};
-  }
-  // Otherwise, return the data.
-  return {
-    pageData,
-    issuesData
-  };
-};