@jrpool/kilotest 46.0.7 → 46.0.11

package/docs/SERVICE.md

@@ -203,7 +203,7 @@ kilotest.com {
   }
   @allowedPOST {
     method POST
-    path /mcp /requestRetest.html/* /requestTest.html /recAction.html /reannotate.html /renewWCAG.html /api/* /tutorialComment.html
+    path /mcp /requestRetest.html/* /requestTest.html /recAction.html /reannotate.html /renewWCAG.html /api/* /tutorialComment.html /worker/job /worker/report
   }
   # Respond to OPTIONS requests.
   @allowedOPTIONS method OPTIONS
@@ -259,6 +259,30 @@ When a new version of the `kilotest` package has been published, the service can
 1. Update the Playwright browsers: `npx playwright install`
 1. Restart the service: `pm2 restart kilotest`
 
+## Branch protection
+
+The `main` branch on GitHub is protected to prevent direct pushes and to require a passing smoke test before any pull request can merge. The settings are configured in the GitHub repository as a ruleset under Settings > Branches > Branch protection rules.
+
+The required settings are:
+
+1. Require a pull request before merging.
+1. Require status checks to pass before merging.
+1. Add "Smoke Test" as a required status check.
+1. Require branches to be up to date before merging.
+1. Do not allow bypassing the above settings.
+
+A copy of the ruleset is located at `docs/Smoke Test.json`.
+
+The smoke-test check is defined in `.github/workflows/smoke-test.yml` and runs `smokeTest.js`, which sends HTTPS requests to all valid GET and POST paths on the deployed server and verifies that Caddy forwards each one to Kilotest rather than returning a bare 404.
+
+When a pull request adds a new route to `index.js`, the Caddyfile at `/etc/caddy/Caddyfile` on the server must be updated to forward the new path before the pull request can merge. The workflow to follow is:
+
+1. Add the new route to the `routes` constant and the dispatch chain in `index.js`.
+1. Update the Caddyfile (`/etc/caddy/Caddyfile`) on the server to forward the new path.
+1. Open or update the pull request.
+1. The smoke test workflow runs and verifies that all paths, including the new one, are forwarded by Caddy.
+1. If the Caddyfile was not updated, the smoke test reports a bare 404 for the new path, the required status check fails, and the pull request cannot merge.
+
 ## Performance
 
 The Cloud Compute host, in initial testing, took about 2.5 as long to process an example job as an Apple M2 Pro MacBook Pro with 16GB of memory. After tuning, the ratio was reduced to about 1.7.

package/index.js

@@ -63,10 +63,42 @@ const answer = {
 
 // CONSTANTS
 
+// Paths that the application is authorized to handle, by method, as glob-style patterns where * matches any sequence of characters.
+const routes = exports.routes = {
+  GET: [
+    '/mcp',
+    '/',
+    '/index.html',
+    '/robots.txt',
+    '/openapi.yaml',
+    '/openapi.json',
+    '/swagger.yaml',
+    '/swagger.json',
+    '/api-docs',
+    '/llms.txt',
+    '/llms-full.txt',
+    '/sitemap.xml',
+    '/style.css',
+    '/fullReport.json/*',
+    '/api/*',
+    '/tutorial/images/*',
+    '/favicon.*',
+    '*.html*'
+  ],
+  POST: [
+    '/mcp',
+    '/requestTest.html',
+    '/requestRetest.html/*',
+    '/recAction.html',
+    '/reannotate.html',
+    '/renewWCAG.html',
+    '/worker/job',
+    '/worker/report',
+    '/api/*',
+    '/tutorialComment.html'
+  ]
+};
 const protocol = process.env.PROTOCOL || 'http';
-const queuePath = () => path.join(jobsPath(), 'queue');
-const claimedPath = () => path.join(jobsPath(), 'claimed');
-const failedPath = () => path.join(jobsPath(), 'failed');
 // Credentials of the Testaro workers, by worker ID, from a JSON-object environment variable.
 // Each worker ID maps to a secret (used only to authenticate the worker, never published) and a
 // name (a non-secret label safe to publish, e.g. in report data and logs).
@@ -89,10 +121,26 @@ const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
 
 // FUNCTIONS
 
+const queuePath = () => path.join(jobsPath(), 'queue');
+const claimedPath = () => path.join(jobsPath(), 'claimed');
+const failedPath = () => path.join(jobsPath(), 'failed');
+// Returns whether a pathname matches a glob-style pattern.
+const matchPath = (pattern, pathname) => {
+  const regex = new RegExp(
+    '^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'
+  );
+  return regex.test(pathname);
+};
+// Returns whether a pathname is authorized for a method.
+const isPathAllowed = exports.isPathAllowed = (method, pathname) => {
+  const patterns = routes[method] || [];
+  return patterns.some(pattern => matchPath(pattern, pathname));
+};
 // Serves or sends an error message.
-const serveError = async (error, response, isHumanUser = true, statusCode = 400) => {
+const serveError = exports.serveError = async (error, response, isHumanUser = true, statusCode = 400) => {
   const errorLines = Object.entries(error).map(pair => `${pair[0]}: ${pair[1]}`);
-  console.log(errorLines.join('\n') || 'ERROR');
+  const errorSummary = errorLines.join('\n') || 'ERROR';
+  console.log(errorSummary);
   if (!response.writableEnded) {
     response.statusCode = statusCode;
     // If the request is from a human user:
@@ -103,7 +151,8 @@ const serveError = async (error, response, isHumanUser = true, statusCode = 400)
       response.setHeader('Access-Control-Allow-Origin', '*');
       response.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=3000');
       const errorTemplate = await fs.readFile('error.html', 'utf8');
-      const errorPage = errorTemplate.replace(/__error__/, error.message || 'ERROR');
+      const errorMessage = error.message || 'ERROR';
+      const errorPage = errorTemplate.replace(/__error__/, errorMessage);
       response.end(errorPage);
     }
     // Otherwise, i.e. if it is from an agent:
@@ -159,21 +208,16 @@ const checkBalancesForAlerts = async report => {
         const balanceData = JSON.parse(balanceJSON);
         // Get an estimate of the balance after this job.
         const newBalance = balanceData.balance - cost;
-        if (typeof newBalance === 'number') {
-          // Update the recorded balance.
-          await fs.writeFile(balancePath, getJSON({balance: newBalance}));
-          console.log(`Estimated new AI Service 0 balance: $${newBalance.toFixed(2)}`);
-          // If it is nearing exhaustion:
-          if (newBalance < AI_SERVICE0_THRESHOLD) {
-            // Alert a manager.
-            await sendAlert(
-              'Kilotest: AI service 0 balance low',
-              `Balance of AI service 0 account (https://console.anthropic.com) only about $${newBalance.toFixed(2)} (about $0.01 used per job)`
-            );
-          }
-        }
-        else {
-          console.log('ERROR: AI service 0 balance is not a number');
+        // Update the recorded balance.
+        await fs.writeFile(balancePath, getJSON({balance: newBalance}));
+        console.log(`Estimated new AI Service 0 balance: $${newBalance.toFixed(2)}`);
+        // If it is nearing exhaustion:
+        if (newBalance < AI_SERVICE0_THRESHOLD) {
+          // Alert a manager.
+          await sendAlert(
+            'Kilotest: AI service 0 balance low',
+            `Balance of AI service 0 account (https://console.anthropic.com) only about $${newBalance.toFixed(2)} (about $0.01 used per job)`
+          );
         }
       }
       catch (error) {
@@ -183,9 +227,11 @@ const checkBalancesForAlerts = async report => {
   }
 };
 // Creates an error object about a suspicious request.
-const getAbuseError = (request, reason) => {
+const getAbuseError = exports.getAbuseError = (request, reason) => {
   const {method, url, headers} = request;
-  const ip = headers['x-forwarded-for'] || request.socket.remoteAddress || 'unknown';
+  const forwardedFor = headers['x-forwarded-for'];
+  const remoteAddress = request.socket.remoteAddress;
+  const ip = forwardedFor || remoteAddress || 'unknown';
   return {
     message: 'Invalid request',
     reason,
@@ -205,13 +251,7 @@ const getBasicAuth = request => {
   if (!match) {
     return null;
   }
-  let decoded;
-  try {
-    decoded = Buffer.from(match[1], 'base64').toString('utf8');
-  }
-  catch {
-    return null;
-  }
+  const decoded = Buffer.from(match[1], 'base64').toString('utf8');
   const sepIndex = decoded.indexOf(':');
   if (sepIndex === -1) {
     return null;
@@ -321,8 +361,12 @@ const requestHandler = async (request, response) => {
   const pathTail = pathname.split('/').slice(2).join('/');
   // If the request is a GET request:
   if (method === 'GET') {
+    // If the path is not authorized for GET requests:
+    if (!isPathAllowed('GET', pathname)) {
+      await serveError({message: `ERROR: Invalid GET request (${pathname})`}, response, true);
+    }
     // If it is for the model context protocol server:
-    if (pathname === mcpPath) {
+    else if (pathname === mcpPath) {
       // Handle the MCP request.
       await handleMCP(request, response);
     }
@@ -516,7 +560,8 @@ const requestHandler = async (request, response) => {
           '.webp': 'image/webp',
           '.svg': 'image/svg+xml'
         };
-        setHeaders(mimeTypes[ext] || 'application/octet-stream', null, 'low');
+        const mimeType = mimeTypes[ext] || 'application/octet-stream';
+        setHeaders(mimeType, null, 'low');
         response.end(img);
       }
       catch {
@@ -554,8 +599,12 @@ const requestHandler = async (request, response) => {
   }
   // Otherwise, if the request is a POST request:
   else if (method === 'POST') {
+    // If the path is not authorized for POST requests:
+    if (!isPathAllowed('POST', pathname)) {
+      await serveError({message: 'ERROR: Invalid POST request'}, response, true);
+    }
     // If it is for the model context protocol server:
-    if (pageName === 'mcp') {
+    else if (pageName === 'mcp') {
       await handleMCP(request, response);
     }
     // Otherwise, i.e. if it is not for the MCP server:
@@ -731,8 +780,9 @@ const requestHandler = async (request, response) => {
           // Otherwise, if it is report acquisition:
           else if (service === 'report') {
             const {report} = postData;
-            const {id, target} = report;
-            const {what, url} = target;
+            const reportObj = report || {};
+            const {id, target} = reportObj;
+            const {what, url} = target || {};
             const [timeStamp, jobID] = id?.split('-') ?? ['', ''];
             // If the request is syntactically valid:
             if (id && isTimeStamp(timeStamp) && isJobID(jobID) && what && url) {
@@ -778,12 +828,6 @@ const requestHandler = async (request, response) => {
               await serveError({message: 'ERROR: Request invalid'}, response, false);
             }
           }
-          // Otherwise, i.e. if the service is invalid:
-          else {
-            await serveError(
-              {message: 'ERROR: Invalid service request from Testaro worker'}, response, false
-            );
-          }
         }
         // Otherwise, i.e. if it is not authenticated:
         else {
@@ -847,11 +891,6 @@ const requestHandler = async (request, response) => {
           response.end(JSON.stringify({status: 'error', message: answerData.message}));
         }
       }
-      // Otherwise, i.e. if it is any other POST request:
-      else {
-        // Report its invalidity.
-        await serveError({message: 'ERROR: Invalid POST request'}, response, true);
-      }
     }
   }
   // Otherwise, i.e. if it is neither a GET nor a POST request:
@@ -872,36 +911,40 @@ const serve = async (protocolModule, options) => {
   for (const path of [queuePath(), claimedPath(), failedPath(), hiddenReportsPath(), reportsPath()]) {
     await fs.mkdir(path, {recursive: true});
   }
-  const server = protocolModule === 'https'
+  const server = protocolModule === https
     ? https.createServer(options, requestHandler)
     : http.createServer(requestHandler);
   const port = process.env.PORT || '3000';
   server.listen(port, () => {
     console.log(`Kilotest server listening at ${protocol}://localhost:${port}.`);
   });
+  return server;
 };
 
-// EXECUTION
+exports.serve = serve;
 
-if (require.main === module) {
-  if (protocol === 'http') {
+// Starts the server using the configured protocol and credentials.
+exports.startServer = async () => {
+  const startProtocol = process.env.PROTOCOL || 'http';
+  if (startProtocol === 'http') {
     console.log('Starting HTTP server');
-    serve(http, {});
+    return serve(http, {});
   }
-  else if (protocol === 'https') {
+  else if (startProtocol === 'https') {
     console.log('Starting HTTPS server');
-    fs.readFile(process.env.KEY, 'utf8')
-    .then(
-      key => {
-        fs.readFile(process.env.CERT, 'utf8')
-        .then(
-          cert => {
-            serve(https, {key, cert});
-          },
-          error => console.log(error.message)
-        );
-      },
-      error => console.log(error.message)
-    );
+    const key = await fs.readFile(process.env.KEY, 'utf8');
+    const cert = await fs.readFile(process.env.CERT, 'utf8');
+    return serve(https, {key, cert});
+  }
+};
+
+// Runs the server if the module was loaded directly (not required by a test).
+exports.runIfMain = (mainModule, currentModule) => {
+  if (mainModule === currentModule) {
+    exports.startServer().catch(error => console.log(error.message));
   }
-}
+};
+
+// EXECUTION
+
+exports.runIfMain(require.main, module);