@jrpool/kilotest 42.0.5 → 43.0.0

package/diagnoses/index.js

@@ -15,7 +15,7 @@ const {
   isHidden,
   tools
 } = require('../util');
-const {issues} = require('testilo/procs/score/tic');
+const {issues: issueSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -62,7 +62,7 @@ const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, qu
   query.target = what;
   query.urlLink = urlLink;
   query.testInfo = testInfo;
-  query.issue = issues[issueID]?.summary;
+  query.issue = issueSpecs[issueID]?.summary;
   // If adding the issue summary failed:
   if (!query.issue) {
     // Populate the query with the reason.
@@ -71,7 +71,7 @@ const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, qu
     return;
   }
   // Otherwise, i.e. if it succeeded, get the issue details.
-  const issue = issues[issueID];
+  const issue = issueSpecs[issueID];
   const {wcag, weight, why} = issue;
   query.why = why;
   query.priority = getWeightName(weight);

package/index.js

@@ -26,8 +26,7 @@ const {
   jobsPath,
   makeReportsExtract,
   recsLock,
-  reportsPath,
-  ruleIDs
+  reportsPath
 } = require('./util');
 const {handleMCP, mcpPath} = require('./mcp');
 const fs = require('fs/promises');
@@ -750,7 +749,7 @@ const requestHandler = async (request, response) => {
                 // Update the data on the available reports.
                 await makeReportsExtract();
                 // Annotate the report.
-                await annotateReport(ruleIDs, timeStamp, jobID);
+                await annotateReport(timeStamp, jobID);
                 console.log(`Testaro report ${id} was annotated, saved, and indexed`);
                 // Check the monetary balances and send alerts if nearing exhaustion.
                 await checkBalancesForAlerts(report);

package/issues/index.js

@@ -14,7 +14,7 @@ const {
   getWeightName,
   objectSort,
 } = require('../util');
-const {issues} = require('testilo/procs/score/tic');
+const {issues: issueSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -69,13 +69,13 @@ const getIssuesSummary = async () => {
   Object.entries(issuesData).forEach(([issueID, data]) => {
     const {count, reporters} = data;
     // If the issue is still classified:
-    if (issues[issueID]) {
+    if (issueSpecs[issueID]) {
       // Increment the report violation count by the issue violation count.
       summary.totalCount += count;
       // Add the issue data and an initilized percentage to the summary.
       summary.issues.push({
         issueID,
-        weight: issues[issueID].weight,
+        weight: issueSpecs[issueID].weight,
         count,
         percentage: 0,
         reporters: getToolNamesString(reporters)
@@ -129,7 +129,7 @@ const populateQuery = async query => {
       if (reportedIssue.weight === weight && percentage >= 2) {
         existsIssue = true;
         // Get the data on it from the issue classification.
-        const issue = issues[issueID];
+        const issue = issueSpecs[issueID];
         const {summary, wcag, why} = issue;
         const wcagLink = `<a href="${getWCAGLink(wcag)}">${wcag}</a>`;
         // Add a description of it to the lines.

package/llms-full.txt

@@ -174,6 +174,6 @@ More information about Kilotest features and internals:
 - [Tutorial on authorizing language models to use Kilotest](https://kilotest.com/qai/)
 - [README file](https://github.com/jrpool/kilotest/blob/main/README.md)
 - [Testaro rule-engine orchestrator](https://github.com/jrpool/testaro)
-- [Testilo rule classifier](https://github.com/jrpool/testilo/blob/main/procs/score/tic.js)
+- [Testaro rule classifier](https://github.com/jrpool/testaro-issues)
 - [Research report on Testaro](https://arxiv.org/abs/2309.10167)
 - [Blog post on the Kilotest use case](https://medium.com/cvs-health-tech-blog/how-to-run-a-thousand-accessibility-tests-63692ad120c3)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "42.0.5",
+  "version": "43.0.0",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "engines": {
@@ -35,6 +35,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "*",
     "dotenv": "*",
+    "testaro-issues": "*",
     "zod": "*"
   },
   "devDependencies": {

package/reannotate/index.js

@@ -5,7 +5,7 @@
 
 // IMPORTS
 
-const {annotateReport, getReportExtracts, ruleIDs} = require('../util');
+const {annotateReport, getReportExtracts} = require('../util');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -23,7 +23,7 @@ exports.answer = async authCode => {
       for (const reportExtract of reportExtracts) {
         const {timeStamp, jobID} = reportExtract;
         // Reannotate it.
-        const annotationError = await annotateReport(ruleIDs, timeStamp, jobID);
+        const annotationError = await annotateReport(timeStamp, jobID);
         // If this failed:
         if (annotationError) {
           // Return an error page.

package/reannotateForm/index.js

@@ -5,7 +5,7 @@
 
 // IMPORTS
 
-const {getIssue, getReport, getReportExtracts, ruleIDs} = require('../util');
+const {getIssue, getReport, getReportExtracts} = require('../util');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -40,7 +40,7 @@ const populateQuery = async query => {
         result.standardResult.instances.forEach(instance => {
           const {ruleID} = instance;
           // Get the issue ID of the rule, or null if none.
-          const issueID = getIssue(ruleIDs, which, ruleID);
+          const issueID = getIssue(which, ruleID);
           // If the issue ID of the instance differs from that of the rule:
           if ((instance.issueID || null) !== issueID) {
             // Add the rule and the report to the rules with changed issue IDs.
@@ -88,21 +88,20 @@ const populateQuery = async query => {
   });
   // Add the lines to the query.
   query.reClassified = reClassifiedLines.join('\n');
+  const formLines = [];
   if (reClassifiedLines.length) {
     query.how = 'Each <q>reclassified</q> rule indicates that report annotations are out of date. To update them, submit your authorization code.';
-    const formLines = [];
-    formLines.push(`${margin}<form action="/reannotate.html" method="post">`);
-    formLines.push(
-      `${margin}  <p><label>Authorization code: <input size="3" minLength="3" maxlength="3" name="authCode" required></label></p>`
-    );
-    formLines.push(`${margin}  <p><button type="submit">Reannotate</button></p>`);
-    formLines.push(`${margin}</form>`);
-    query.reannotateForm = formLines.join('\n');
   }
   else {
-    query.how = 'No violated rules have been classified or reclassified after being reported, so reannotation of the reports is not necessary.';
-    query.reannotateForm = '';
+    query.how = 'No violated rules have been classified or reclassified after being reported, so reannotation of the reports is normally not necessary. If there is another reason for reannotation, submit your authorization code.';
   }
+  formLines.push(`${margin}<form action="/reannotate.html" method="post">`);
+  formLines.push(
+    `${margin}  <p><label>Authorization code: <input size="3" minLength="3" maxlength="3" name="authCode" required></label></p>`
+  );
+  formLines.push(`${margin}  <p><button type="submit">Reannotate</button></p>`);
+  formLines.push(`${margin}</form>`);
+  query.reannotateForm = formLines.join('\n');
 };
 // Returns a page disclosing newly classified rules and a form to reannotate reports.
 exports.answer = async () => {

package/reportIssue/index.js

@@ -17,7 +17,7 @@ const {
   isHidden,
   makeBreakable,
 } = require('../util');
-const {issues} = require('testilo/procs/score/tic');
+const {issues: issueSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -35,7 +35,7 @@ const populateQuery = async (issueID, timeStamp, jobID, query) => {
     return;
   }
   // Otherwise, i.e. if it succeeded, add the issue summary to the query.
-  query.issue = issues[issueID]?.summary;
+  query.issue = issueSpecs[issueID]?.summary;
   // If adding the issue summary failed:
   if (!query.issue) {
     // Populate the query with the reason.
@@ -48,7 +48,7 @@ const populateQuery = async (issueID, timeStamp, jobID, query) => {
   query.target = what;
   query.urlLink = urlLink;
   query.testInfo = testInfo;
-  const issue = issues[issueID];
+  const issue = issueSpecs[issueID];
   const {wcag, weight, why} = issue;
   query.why = why;
   query.priority = getWeightName(weight);

package/reportIssues/index.js

@@ -18,7 +18,7 @@ const {
   objectSort,
   tools
 } = require('../util');
-const issuesClassification = require('testilo/procs/score/tic').issues;
+const {issues: issueSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -67,7 +67,7 @@ const getIssuesData = async (timeStamp, jobID) => {
           const {catalogIndex, issueID} = instance;
           // If the instance identifies its rule as belonging to a non-ignorable issue:
           if (issueID && issueID !== 'ignorable') {
-            const issueClassification = issuesClassification[issueID];
+            const issueClassification = issueSpecs[issueID];
             // If the issue has a current weighted classification:
             if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
               const {summary, wcag, weight, why} = issueClassification;

package/rules/index.html

@@ -20,9 +20,7 @@
         <li>Related WCAG standard: __wcag__</li>
       </ul>
       <h2>Rules</h2>
-      <ul>
-        __rules__
-      </ul>
+      __rules__
     </main>
   </body>
 </html>

package/rules/index.js

@@ -8,9 +8,9 @@
 const {
   getWeightName,
   htmlSafe,
-  tools
+  ruleEngines
 } = require('../util');
-const {issues} = require('testilo/procs/score/tic');
+const {issues: issueSpecs, issueRules, rules: ruleSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -19,8 +19,12 @@ const path = require('path');
 // Adds parameters to a query for the answer page.
 const populateQuery = async (issueID, query) => {
   // Add facts about the issue to the query.
-  query.issue = issues[issueID].summary;
-  const issue = issues[issueID];
+  query.issue = issueSpecs[issueID]?.summary;
+  if (!query.issue) {
+    query.error = 'Issue not found';
+    return;
+  }
+  const issue = issueSpecs[issueID];
   const {wcag, weight, why} = issue;
   query.why = why;
   query.priority = getWeightName(weight);
@@ -28,44 +32,61 @@ const populateQuery = async (issueID, query) => {
   // Initialize the lines.
   const lines = [];
   const margin = ' '.repeat(6);
-  // For each tool with any rules belonging to the issue:
-  Object.keys(issue.tools).forEach(toolID => {
-    // Add a line.
-    lines.push(`${margin}<li><h3>${tools[toolID][0]} rules</h3>`);
-    lines.push(`${margin}  <ul>`);
-    const tool = issue.tools[toolID];
-    // For each rule of the tool belonging to the issue:
-    Object.keys(tool).forEach(ruleID => {
-      const rule = tool[ruleID];
-      const {what} = rule;
-      // Add facts about the rule.
-      if (what === ruleID) {
-        lines.push(`${margin}    <li><code>${htmlSafe(ruleID)}</code></li>`);
-      }
-      else {
-        lines.push(`${margin}    <li><code>${htmlSafe(ruleID)}</code>: ${htmlSafe(what)}</li>`);
+  // For each rule engine with any rules belonging to the issue:
+  Object.keys(issueRules[issueID] ?? {}).forEach(engineID => {
+    // Add a heading for the rules of the rule engine.
+    lines.push(`${margin}<h3>${ruleEngines[engineID][0]} rules</h3>`);
+    const {invariant, variable} = issueRules[issueID][engineID];
+    const rulesByType = {
+      invariant,
+      variable
+    };
+    // For each rule variability:
+    Object.keys(rulesByType).forEach(typeName => {
+      // If any rules of the rule engine belonging to the issue have it:
+      if (rulesByType[typeName].length) {
+        // Add a heading for them.
+        lines.push(`${margin}<h4>${typeName} rules</h4>`);
+        // Add a list of facts about them.
+        lines.push(`${margin}<ul>`);
+        rulesByType[typeName].forEach(ruleID => {
+          const {what} = ruleSpecs[engineID][typeName][ruleID];
+          if (what === ruleID) {
+            lines.push(`${margin}  <li><code>${htmlSafe(ruleID)}</code></li>`);
+          }
+          else {
+            lines.push(`${margin}  <li><code>${htmlSafe(ruleID)}</code>: ${htmlSafe(what)}</li>`);
+          }
+        });
+        lines.push(`${margin}</ul>`);
       }
     });
-    lines.push(`${margin}  </ul>`);
-    lines.push(`${margin}</li>`);
   });
   // Add the lines to the query.
   query.rules = lines.join('\n');
 };
-// Returns a page answering the violators question.
+// Returns a page answering the issue-rules question.
 exports.answer = async issueID => {
   const query = {};
   // Create a query to replace the placeholders.
   await populateQuery(issueID, query);
-  // Get the template.
-  let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
-  // Replace its placeholders.
-  Object.keys(query).forEach(param => {
-    answerPage = answerPage.replace(new RegExp(`__${param}__`, 'g'), query[param]);
-  });
-  // Return the populated page.
+  // If this succeeded:
+  if (query.issue) {
+    // Get the template.
+    let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+    // Replace its placeholders.
+    Object.keys(query).forEach(param => {
+      answerPage = answerPage.replace(new RegExp(`__${param}__`, 'g'), query[param]);
+    });
+    // Return the populated page.
+    return {
+      status: 'ok',
+      answerPage
+    };
+  }
+  // Otherwise, i.e. if it failed, report this.
   return {
-    status: 'ok',
-    answerPage
+    status: 'error',
+    message: query.error
   };
 };

package/util.js

@@ -6,9 +6,7 @@
 // IMPORTS
 
 const {sendAlert} = require('./alerts');
-const issuesClassification
-= exports.issuesClassification
-= require('testilo/procs/score/tic').issues;
+const {issues: issueSpecs, rules: ruleSpecs} = require('testaro-issues');
 const fs = require('fs/promises');
 const path = require('path');
 const querystring = require('querystring');
@@ -120,26 +118,25 @@ const getDateTime = exports.getDateTime = timeStamp => {
   return dateTime.toString() === 'Invalid Date' ? null : dateTime;
 };
 // Returns the issue that a rule belongs to, or null if none.
-const getIssue = exports.getIssue = (ruleIDs, toolID, ruleID) => {
-  const {invariant, variable} = ruleIDs;
-  // Initialize the issue ID of the rule as if the rule ID is invariant.
-  let issueID = invariant[toolID]?.[ruleID];
-  // If the initialization succeeded:
-  if (issueID) {
-    // Return it.
-    return issueID;
+const getIssue = exports.getIssue = (engineID, ruleID) => {
+  const engineRules = ruleSpecs[engineID];
+  // If the rule engine has no rule specifications:
+  if (!engineRules) {
+    // Return a failure result.
+    return null;
   }
-  // Otherwise, change the rule ID to the first matching variable rule ID of the tool.
-  ruleID = Object
-  .keys(variable[toolID] ?? {})
-  .find(variableRuleID => new RegExp(variableRuleID).test(ruleID));
-  // If the change succeeded:
-  if (ruleID) {
-    // Return the issue ID.
-    return variable[toolID][ruleID];
+  const {invariant, variable} = engineRules;
+  // If the rule ID is invariant and classified:
+  if (invariant[ruleID]) {
+    // Return its issue ID.
+    return invariant[ruleID].issueID;
   }
-  // Otherwise, i.e. if no issue was found, return a failure result.
-  return null;
+  // Otherwise, find the first matching variable rule ID pattern.
+  const variableRuleID = Object
+  .keys(variable)
+  .find(pattern => new RegExp(`^${pattern}$`).test(ruleID));
+  // Return the issue ID if a pattern matched, or a failure result otherwise.
+  return variableRuleID ? variable[variableRuleID].issueID : null;
 };
 // Gets the names and categories of the job files.
 const getJobNames = exports.getJobNames = async () => {
@@ -179,58 +176,6 @@ const getObject = exports.getObject = async filePath => {
 exports.getRandomString = length => {
   return Math.random().toString(36).slice(2, length + 2);
 };
-// Compiles a directory of the issue classifications of invariant and variable rules.
-const getRuleIDs = exports.getRuleIDs = () => {
-  // Initialize data on invariant and variable rule IDs.
-  const invariant = {};
-  const variable = {};
-  // Initialize a validity checker.
-  const validityChecker = {};
-  // For each classified issue:
-  Object.keys(issuesClassification).forEach(issueID => {
-    const {tools, weight} = issuesClassification[issueID];
-    // If the weight is invalid:
-    if (weight < 1 || weight > 4) {
-      // Report this.
-      console.log(`ERROR: Issue ${issueID} weight is invalid`);
-    }
-    // For each tool that has any rules belonging to the issue:
-    Object.keys(tools).forEach(toolID => {
-      // For each such rule:
-      Object.keys(tools[toolID]).forEach(ruleID => {
-        // If it is a duplicate:
-        if (validityChecker[toolID]?.has(ruleID)) {
-          // Report this.
-          console.log(`ERROR: Rule ${ruleID} of tool ${toolID} belongs to 2 issues`);
-        }
-        // Otherwise, i.e. if it is not a duplicate:
-        else {
-          // Add it to the classified rules of the tool.
-          validityChecker[toolID] ??= new Set();
-          validityChecker[toolID].add(ruleID);
-        }
-        const rule = tools[toolID][ruleID];
-        // If it is variable:
-        if (rule.variable) {
-          variable[toolID] ??= {};
-          // Add its ID and the issue ID to the variable rule IDs.
-          variable[toolID][ruleID] = issueID;
-        }
-        // Otherwise, i.e. if it is invariant:
-        else {
-          invariant[toolID] ??= {};
-          // Add its ID and the issue ID to the invariant rule IDs.
-          invariant[toolID][ruleID] = issueID;
-        }
-      });
-    });
-  });
-  // Return the data.
-  return {
-    invariant,
-    variable
-  };
-};
 // Returns a time stamp from a date.
 const getTimeStamp = exports.getTimeStamp = date => {
   const timeStamp = date.toISOString().slice(2).replace(/[-:]/g, '').slice(0, 11);
@@ -449,8 +394,6 @@ exports.processRec = async (testType, dirName, what, url, why) => {
 };
 // Concurrency lock for the `recs.json` file.
 const recsLock = exports.recsLock = createLock();
-// Variable and invariant rules.
-exports.ruleIDs = getRuleIDs();
 // Updates the test recommendations as a transaction.
 const updateRecs = exports.updateRecs = (what, url, why) => recsLock(async () => {
   // Get the data on waiting recommendations.
@@ -525,7 +468,7 @@ const getReport = exports.getReport = async (timeStamp, jobID) => {
   }
 };
 // Adds issue IDs to the standard instances of a report.
-exports.annotateReport = async (ruleIDs, timeStamp, jobID) => {
+exports.annotateReport = async (timeStamp, jobID) => {
   // Get a copy of the report.
   const report = await getReport(timeStamp, jobID);
   // If this failed:
@@ -544,7 +487,7 @@ exports.annotateReport = async (ruleIDs, timeStamp, jobID) => {
       for (const instance of result?.standardResult?.instances ?? []) {
         const {ruleID} = instance;
         // Classify its rule.
-        const issueID = getIssue(ruleIDs, which, ruleID);
+        const issueID = getIssue(which, ruleID);
         // If the rule was classifiable:
         if (issueID) {
           // Add the issue ID to the instance.
@@ -617,7 +560,7 @@ exports.getReportData = async (timeStamp, jobID) => {
       instances.forEach(instance => {
         const {catalogIndex, issueID} = instance;
         // If it has a non-ignorable classified issue ID:
-        if (issueID && issuesClassification[issueID] && issueID !== 'ignorable') {
+        if (issueID && issueSpecs[issueID] && issueID !== 'ignorable') {
           // Ensure that the tool is in the temporary data.
           reporterIDSet.add(which);
           // Ensure that the issue is in the temporary data.