@jrpool/kilotest 40.0.0 → 41.0.0

package/.devin/config.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Exec(git status)"
+    ]
+  }
+}

package/README.md

@@ -18,7 +18,7 @@ Testing for Kilotest is performed by one or more testing agents that obtain jobs
 
 An active production instance of Kilotest may require multiple testing agents to handle the load, because testing one web page typically takes about 3 minutes and agents test only one page at a time.
 
-## Getting started locally with 1 testing agent
+## Getting started locally with 1 testing worker
 
 ### Installation
 
@@ -30,7 +30,7 @@ In the steps below, hosts `T` and `K` may be the same host or two different host
 1. Clone the Kilotest repository into a new directory on host `K`.
 1. In that directory, install the Kilotest dependencies: `npm install`.
 1. Copy the `env.testaro` file from the `kilotest` directory to `.env` in the `testaro` directory and replace the `__placeholder__` values in `.env` with actual values.
-1. Copy the `env.example` file in the `kilotest` directory to a new `.env` file in the same directory and replace the `__placeholder__` values in `.env` with actual values.
+1. Copy the `env.example` file in the `kilotest` directory to a new `.env` file in the same directory and replace the `__placeholder__` values in `.env` with actual values. `TESTARO_WORKERS` is a JSON object mapping each Testaro worker's ID to an object with its `secret` and its `name`; the ID and secret for a worker must match the `NETWATCH_URL_WORKER_ID` and `NETWATCH_URL_AUTH` values in that worker's `.env`. A worker authenticates each request with an HTTP Basic `Authorization` header (`id:secret`, base64-encoded); as of this writing that header is not yet sent by the published Testaro package, so this scheme requires a matching update to Testaro's `netWatch.js` request code (only to send the header; no change to Testaro's handling of jobs or reports is needed). The ID is a credential and is never published; the `name` is written by Kilotest into a job's and report's `sources.worker` property, which Testaro does not read or alter, so it is a distinct property from `sources.agent` (Testaro's own, untrusted, self-reported label, which Kilotest ignores). Choose a `name` that is safe to appear in every report Kilotest serves.
 
 ### Usage
 

package/alerts.js

@@ -28,6 +28,7 @@ exports.sendAlert = (subject, body) => new Promise(resolve => {
       subject,
       text: body
     });
+    // Ask the alerting host to send an alert to the manager.
     const req = https.request({
       hostname: ALERT_API_HOST,
       path: ALERT_API_PATH,
@@ -43,10 +44,14 @@ exports.sendAlert = (subject, body) => new Promise(resolve => {
         data += chunk;
       });
       res.on('end', () => {
+        // If this succeeded:
         if (res.statusCode >= 200 && res.statusCode < 300) {
+          // Report this.
           console.log(`Alert sent (${subject})`);
         }
+        // Otherwise, i.e. if it failed:
         else {
+          // Report this.
           console.log(`ERROR: Alert API responded ${res.statusCode}: ${data}`);
         }
         resolve();

package/docs/SERVICE.md

@@ -203,7 +203,7 @@ kilotest.com {
   }
   @allowedPOST {
     method POST
-    path /mcp /retestRec.html/* /testRec.html /recAction.html /reannotate.html /wcagRenew.html /api/* /tutorialComment.html
+    path /mcp /retestRec.html/* /testRec.html /recAction.html /reannotate.html /wcagRenew.html /api/* /worker/* /tutorialComment.html
   }
   # Respond to OPTIONS requests.
   @allowedOPTIONS method OPTIONS
@@ -244,7 +244,7 @@ kilotest.com {
 
 This configuration can be futher tightened if experience warrants.
 
-The `flush_interval` setting prevents granular reporting by Testaro agents from being buffered, so the updates reach the browser without delay.
+The `flush_interval` setting prevents granular reporting by Testaro workers from being buffered, so the updates reach the browser without delay.
 
 ## Version management
 
@@ -326,7 +326,7 @@ After that the maintainer can inspect the original and restored directories and
 
 ### Possible future Testaro integration
 
-Kilotest uses Testaro to run jobs. In previous versions of Kilotest, Testaro was a dependency. It is currently not a dependenc. Instead, Testaro instances are installed on one or more other hosts, and each instance polls Kilotest to ask for jobs to run.
+Kilotest uses Testaro to run jobs. In previous versions of Kilotest, Testaro was a dependency. It is currently not a dependency. Instead, Testaro instances are installed on one or more other hosts, and each instance polls Kilotest to ask for jobs to run.
 
 In case Testaro again becomes a dependency of Kilotest, the notes below on security issues will be useful.
 

package/env.example

@@ -1,32 +1,29 @@
-# ENVIRONMENT=test for a dev/test host, =production for a deployed host.
+# test (local) or production (deployed host).
 ENVIRONMENT=__placeholder__
-# PROTOCOL=http for a local dev/test host or reverse-proxied server, otherwise =https.
+# http (local or reverse-proxied server) or https.
 PROTOCOL=__placeholder__
-# AUTH_CODE for manager operations; must be 3 characters long.
+# 3-character password for manager operations.
 AUTH_CODE=__placeholder__
-# Name of Testaro agent.
-TESTARO_AGENT=testaro-agent
-# Testaro agent password (same as NETWATCH_AUTH in the Testaro .env.
-TESTARO_AGENT_PW=__placeholder__
-# Name of research agent (AI agent calling Kilotest as a tool).
-RESEARCH_AGENT=research-agent
-# Research agent password.
-RESEARCH_AGENT_PW=__placeholder__
-# URL of this Kilotest host.
+# ID of each Testaro worker and, for each, its secret and published name.
+TESTARO_WORKERS={"__placeholder__":{"secret":"__placeholder__","name":"__placeholder__"}}
+# URL of this host.
 THIS_KILOTEST_HOST=__placeholder__
-# URL of a dev/test Kilotest host.
+# URL of a dev/test host.
 LOCAL_KILOTEST_HOST=http://localhost:3000
-# URL of the deployed Kilotest host.
+# URL of the deployed host.
 DEPLOYED_KILOTEST_HOST=https://kilotest.com
-# Alert email configuration:
+
+# Alert configuration:
+
 MANAGER_EMAIL=__placeholder__
 ALERT_API_HOST=api.resend.com
 ALERT_API_PATH=/emails
 ALERT_API_KEY=__placeholder__
 ALERT_FROM=__placeholder__
-# Alert thresholds:
-WAVE_BALANCE_THRESHOLD=30
-AI_SERVICE0_BALANCE_THRESHOLD=10
+# Threshold in credits.
+WAVE_BALANCE_THRESHOLD=__placeholder__
+# Threshold in dollars.
+AI_SERVICE0_BALANCE_THRESHOLD=__placeholder__
 # Per-million-token prices (Claude Haiku 4.5) on 2026-05-01 were $1 in, $5 out.
 AI_MODEL0_INPUT_PRICE=0.000001
 AI_MODEL0_OUTPUT_PRICE=0.000005

package/env.testaro

@@ -1,17 +1,43 @@
 # You can get a WAVE API key at https://wave.webaim.org/api.
 WAVE_KEY=__placeholder__
-AGENT=testaro-agent
+# You can get an Anthropic API key at https://console.anthropic.com/.
+ANTHROPIC_API_KEY=__placeholder__
+# Any message from this instance to the server (optional).
+AGENT=__placeholder__
+# Whether to make the browser visible (normally false).
 HEADED_BROWSER=false
+# Whether to output even forked-job logging to the primary log (normally false).
 DEBUG=false
+# Number of seconds to wait between actions (normally 0).
 WAITS=0
+# See https://nodejs.org/api/cli.html#options for all permitted options.
 NODE_OPTIONS='--trace-uncaught --trace-warnings'
+# Whether to suppress a routine warning from QualWeb.
 PUPPETEER_DISABLE_HEADLESS_WARNING=true
-# Replace the next 2 placeholders with:
-# http://localhost:3000 if Kilotest and Testaro are on the same host.
-# the URL of the Kilotest server (e.g., https://kilotest.com) if they are on different hosts.
-NETWATCH_JOB=__placeholder__/api/testaro-agent/job
-NETWATCH_REPORT=__placeholder__/api/testaro-agent/report
-# NETWATCH_AUTH must be identical to TESTARO_AGENT_PW in the Kilotest .env.
-NETWATCH_AUTH=__placeholder__
+# Directory (relative to project root) to watch for jobs.
+JOBDIR=__placeholder__
+# Directory (relative to project root) to write reports to when watching a directory for jobs.
+REPORTDIR=__placeholder__
+# Multiplier for time limits (normally 1).
 TIMEOUT_MULTIPLIER=1
-TMPDIRNAME=scratch
+# Whether to abort the job when launch retries are exhausted or a test act crashes (default false).
+ABORT_ASSERTIVELY=false
+# URL of any non-default (e.g., self-hosted) Nu Html Checker API (see ghcr.io/validator/validator).
+TESTARO_NU_URL=__placeholder__
+# Whether to launch the Playwright chromium browser without its sandbox (normally false).
+# The sandbox requires unprivileged user-namespace cloning, which default container
+# seccomp policies and some hardened hosts prohibit. Set this to true in containers,
+# or run containers with a seccomp profile that permits user-namespace cloning.
+TESTARO_CHROMIUM_NO_SANDBOX=false
+
+# netWatch variables
+
+# Replace the next 2 placeholders with the hostname of the server (e.g., http://localhost:3000 or  https://kilotest.com).
+# URL to poll for available jobs.
+NETWATCH_URL_JOB=__placeholder__/worker/job
+# URL to send a job report.
+NETWATCH_URL_REPORT=__placeholder__/worker/report
+
+# The server requires an authorization header. Identifier and secret of this worker.
+WORKER_ID=__placeholder__
+WORKER_SECRET=__placeholder__

package/index.js

@@ -69,8 +69,18 @@ const protocol = process.env.PROTOCOL || 'http';
 const queuePath = path.join(jobsPath, 'queue');
 const claimedPath = path.join(jobsPath, 'claimed');
 const failedPath = path.join(jobsPath, 'failed');
-const testaroAgent = process.env.TESTARO_AGENT;
-const testaroAgentPW = process.env.TESTARO_AGENT_PW;
+// Credentials of the Testaro workers, by worker ID, from a JSON-object environment variable.
+// Each worker ID maps to a secret (used only to authenticate the worker, never published) and a
+// name (a non-secret label safe to publish, e.g. in report data and logs).
+const workerCredentials = (() => {
+  try {
+    return JSON.parse(process.env.TESTARO_WORKERS || '{}');
+  }
+  catch (error) {
+    console.error(`ERROR: TESTARO_WORKERS is not valid JSON (${error.message})`);
+    return {};
+  }
+})();
 // Values that may require alerts.
 const balancePath = path.join(__dirname, 'ai0Balance.json');
 const jobLock = createLock();
@@ -82,11 +92,11 @@ const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
 // FUNCTIONS
 
 // Serves or sends an error message.
-const serveError = async (error, response, isHumanUser = true) => {
+const serveError = async (error, response, isHumanUser = true, statusCode = 400) => {
   const errorLines = Object.entries(error).map(pair => `${pair[0]}: ${pair[1]}`);
   console.log(errorLines.join('\n') || 'ERROR');
   if (!response.writableEnded) {
-    response.statusCode = 400;
+    response.statusCode = statusCode;
     // If the request is from a human user:
     if (isHumanUser) {
       // Serve an HTML page containing the message property of the error.
@@ -189,19 +199,53 @@ const getAbuseError = (request, reason) => {
     time: new Date().toISOString()
   };
 };
-// Processes a job request from a Testaro agent.
-const processJobRequest = async (request, response, agentID) => jobLock(async () => {
+// Gets the ID and secret from a request's HTTP Basic Authorization header, or null if the header
+// is absent or malformed.
+const getBasicAuth = request => {
+  const header = request.headers['authorization'] || '';
+  const match = header.match(/^Basic\s+(\S+)$/i);
+  if (!match) {
+    return null;
+  }
+  let decoded;
+  try {
+    decoded = Buffer.from(match[1], 'base64').toString('utf8');
+  }
+  catch {
+    return null;
+  }
+  const sepIndex = decoded.indexOf(':');
+  if (sepIndex === -1) {
+    return null;
+  }
+  return {id: decoded.slice(0, sepIndex), secret: decoded.slice(sepIndex + 1)};
+};
+// Gets the published name of the Testaro worker that made a request, or null if it is not
+// authenticated. The ID authenticates the worker and is never published; the name is a
+// non-secret label safe to write into job/report data and log messages.
+const getAuthorizedWorkerName = request => {
+  const credentials = getBasicAuth(request);
+  if (credentials) {
+    const worker = workerCredentials[credentials.id];
+    if (worker && worker.secret === credentials.secret && worker.name) {
+      return worker.name;
+    }
+  }
+  return null;
+};
+// Processes a job request from a Testaro worker.
+const processJobRequest = async (request, response, workerName) => jobLock(async () => {
   let clean = true;
-  const messageStart = `Testaro agent ${agentID} requested a job, `;
+  const messageStart = `Testaro worker ${workerName} requested a job, `;
   const jobNames = await getJobNames();
   const claimedJobNames = jobNames.claimed;
   // For each claimed job:
   for (const jobName of claimedJobNames) {
     const job = await getObject(path.join(jobsPath, 'claimed', jobName));
     const {id, sources} = job;
-    const {agent} = sources;
-    // If its assignee is the agent:
-    if (agent === agentID) {
+    const {worker} = sources;
+    // If its assignee is the worker:
+    if (worker === workerName) {
       const messageEnd = `but has not completed job ${id}`;
       // Report this.
       await serveError({message: `${messageStart}${messageEnd}`}, response, false);
@@ -214,7 +258,7 @@ const processJobRequest = async (request, response, agentID) => jobLock(async ()
       break;
     }
   }
-  // If no aborted-job error was found for the agent:
+  // If no aborted-job error was found for the worker:
   if (clean) {
     const queuedJobNames = jobNames.queue;
     // If any jobs are queued:
@@ -222,18 +266,18 @@ const processJobRequest = async (request, response, agentID) => jobLock(async ()
       const oldestJobName = queuedJobNames[0];
       // Get the first one.
       const firstJob = await getObject(path.join(queuePath, oldestJobName));
-      // Add the agent ID to the job.
-      firstJob.sources.agent = agentID;
+      // Add the public worker name to the job, in a property Testaro does not read or alter.
+      firstJob.sources.worker = workerName;
       console.log(
-        `Job ${firstJob.id} (${firstJob.target.what}) is being sent to the agent.`
+        `Job ${firstJob.id} (${firstJob.target.what}) is being sent to the worker.`
       );
-      // Assign the job to the agent.
+      // Assign the job to the worker.
       response.writeHead(200, {
         'content-type': 'application/json; charset=utf-8'
       });
       response.end(JSON.stringify(firstJob));
       const messageEnd
-      = `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the agent`;
+      = `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the worker`;
       console.log(`${messageStart}${messageEnd}`);
       // Save the job in the claimed-jobs directory.
       await fs.writeFile(
@@ -247,7 +291,7 @@ const processJobRequest = async (request, response, agentID) => jobLock(async ()
       response.writeHead(200, {
         'content-type': 'application/json; charset=utf-8'
       });
-      // Send a no-jobs response to the agent.
+      // Send a no-jobs response to the worker.
       response.end(JSON.stringify({}));
       const messageEnd = 'but no job was in the queue';
       console.log(`${messageStart}${messageEnd}`);
@@ -674,47 +718,67 @@ const requestHandler = async (request, response) => {
           await serveError({message: answerData.message}, response, true);
         }
       }
-      // Otherwise, if it is a request from an agent:
-      else if (pageName === 'api') {
-        // Get the segments of the path after api.
-        const segments = pathTail.split('/');
-        // If the first segment is the ID of the Testaro agent and the agent is authenticated:
-        if (segments[0] === testaroAgent && postData.agentPW === testaroAgentPW) {
-          const agentID = segments[0];
+      // Otherwise, if it is a request from a Testaro worker:
+      else if (pageName === 'worker') {
+        // Authenticate the worker and get its public name.
+        const workerName = getAuthorizedWorkerName(request);
+        // If this succeeded:
+        if (workerName) {
           // Get the requested service from the path.
-          const service = segments[1];
+          const service = pathTail;
           // If the service is job assignment:
           if (service === 'job') {
             // Process the job request as a transaction.
-            await processJobRequest(request, response, agentID);
+            await processJobRequest(request, response, workerName);
           }
-          // Otherwise, if the service is report acquisition:
+          // Otherwise, if it is report acquisition:
           else if (service === 'report') {
             const {report} = postData;
             const {id, target} = report;
             const {what, url} = target;
             const [timeStamp, jobID] = id?.split('-') ?? ['', ''];
-            // If the request is valid:
+            // If the request is syntactically valid:
             if (id && isTimeStamp(timeStamp) && isJobID(jobID) && what && url) {
               const [timeStamp, jobID] = id.split('-');
-              // Save the report.
-              await fs.writeFile(getReportPath(timeStamp, jobID), getJSON(report));
-              // Update the data on the available reports.
-              await makeReportsExtract();
-              // Annotate the report.
-              await annotateReport(ruleIDs, timeStamp, jobID);
-              console.log(`Testaro report ${id} was annotated, saved, and indexed`);
-              // Check the monetary balances and send alerts if nearing exhaustion.
-              await checkBalancesForAlerts(report);
-              // Delete the job.
-              await fs.unlink(path.join(claimedPath, `${id}.json`));
-              console.log(`Completed job ${id} deleted`);
-              // Acknowledge receipt.
-              response.setHeader('content-type', 'application/json; charset=utf-8');
-              response.end(JSON.stringify({status: 'ok'}));
-              console.log(`Testaro report ${id} was received from Testaro agent ${agentID}`);
+              // Get the job the report is from.
+              const claimedJob = await getObject(path.join(claimedPath, `${id}.json`));
+              // If the job was actually assigned to this worker:
+              if (typeof claimedJob === 'object' && claimedJob.sources?.worker === workerName) {
+                console.log(`Testaro report ${id} was received from worker ${workerName}`);
+                // Add the public worker name to the report.
+                report.sources = {...report.sources, worker: workerName};
+                // Save the report.
+                await fs.writeFile(getReportPath(timeStamp, jobID), getJSON(report));
+                // Update the data on the available reports.
+                await makeReportsExtract();
+                // Annotate the report.
+                await annotateReport(ruleIDs, timeStamp, jobID);
+                console.log(`Testaro report ${id} was annotated, saved, and indexed`);
+                // Check the monetary balances and send alerts if nearing exhaustion.
+                await checkBalancesForAlerts(report);
+                // Delete the job.
+                await fs.unlink(path.join(claimedPath, `${id}.json`));
+                console.log(`Completed job ${id} deleted`);
+                // Acknowledge receipt.
+                response.setHeader('content-type', 'application/json; charset=utf-8');
+                response.end(JSON.stringify({status: 'ok'}));
+                console.log(
+                  `Testaro report ${id} was received from Testaro worker ${workerName}`
+                );
+              }
+              // Otherwise, i.e. if the job was not assigned to this worker:
+              else {
+                await serveError(
+                  getAbuseError(
+                    request,
+                    `Worker ${workerName} submitted a report for job ${id}, which was not currently assigned to it`
+                  ),
+                  response,
+                  false
+                );
+              }
             }
-            // Otherwise, i.e. if the request is invalid:
+            // Otherwise, i.e. if the request is syntactically invalid:
             else {
               await serveError({message: 'ERROR: Request invalid'}, response, false);
             }
@@ -722,54 +786,56 @@ const requestHandler = async (request, response) => {
           // Otherwise, i.e. if the service is invalid:
           else {
             await serveError(
-              {message: 'ERROR: Invalid service request from Testaro agent'}, response, false
-            );
-          }
-        }
-        // Otherwise, if the first segment is not the ID of the Testaro agent:
-        else if (segments[0] !== testaroAgent) {
-          // If the service is to receive a test request:
-          if (segments[0] === 'requestTest') {
-            const {description, URL, reason} = postData;
-            // Get the response body.
-            const responseBody = await require(path.join(__dirname, 'api', 'requestTest'))
-            .response([description, URL, reason]);
-            // Send it.
-            setHeaders('application/json', null, 'ultra');
-            response.end(JSON.stringify(responseBody));
-          }
-          // Otherwise, if the service is to receive a retest request:
-          else if (segments[0] === 'requestRetest') {
-            const {reason} = postData;
-            // Get the response body.
-            const responseBody = await require(path.join(__dirname, 'api', 'requestRetest'))
-            .response(segments.slice(1).concat(reason));
-            // Send it.
-            setHeaders('application/json', null, 'ultra');
-            response.end(JSON.stringify(responseBody));
-          }
-          // Otherwise, if the service is to receive a feature request:
-          else if (segments[0] === 'requestFeature') {
-            const {feature} = postData;
-            // Get the response body.
-            const responseBody = await require(path.join(__dirname, 'api', 'requestFeature'))
-            .response([feature]);
-            // Send it.
-            setHeaders('application/json', null, 'ultra');
-            response.end(JSON.stringify(responseBody));
-          }
-          // Otherwise, i.e. if the service is invalid:
-          else {
-            await serveError(
-              {message: 'ERROR: Invalid request'}, response, false
+              {message: 'ERROR: Invalid service request from Testaro worker'}, response, false
             );
           }
         }
-        // Otherwise, i.e. if the request is invalid:
+        // Otherwise, i.e. if it is not authenticated:
         else {
           // Report this.
           await serveError(
-            {message: 'ERROR: Invalid API request'}, response, false
+            {message: 'ERROR: Unauthorized worker request'}, response, false, 401
+          );
+        }
+      }
+      // Otherwise, if it is a request from an API consumer:
+      else if (pageName === 'api') {
+        // Get the segments of the path after api.
+        const segments = pathTail.split('/');
+        // If the service is to receive a test request:
+        if (segments[0] === 'requestTest') {
+          const {description, URL, reason} = postData;
+          // Get the response body.
+          const responseBody = await require(path.join(__dirname, 'api', 'requestTest'))
+          .response([description, URL, reason]);
+          // Send it.
+          setHeaders('application/json', null, 'ultra');
+          response.end(JSON.stringify(responseBody));
+        }
+        // Otherwise, if the service is to receive a retest request:
+        else if (segments[0] === 'requestRetest') {
+          const {reason} = postData;
+          // Get the response body.
+          const responseBody = await require(path.join(__dirname, 'api', 'requestRetest'))
+          .response(segments.slice(1).concat(reason));
+          // Send it.
+          setHeaders('application/json', null, 'ultra');
+          response.end(JSON.stringify(responseBody));
+        }
+        // Otherwise, if the service is to receive a feature request:
+        else if (segments[0] === 'requestFeature') {
+          const {feature} = postData;
+          // Get the response body.
+          const responseBody = await require(path.join(__dirname, 'api', 'requestFeature'))
+          .response([feature]);
+          // Send it.
+          setHeaders('application/json', null, 'ultra');
+          response.end(JSON.stringify(responseBody));
+        }
+        // Otherwise, i.e. if the service is invalid:
+        else {
+          await serveError(
+            {message: 'ERROR: Invalid request'}, response, false
           );
         }
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "40.0.0",
+  "version": "41.0.0",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "scripts": {