@jrpool/kilotest 46.0.7 → 46.0.8

package/.github/workflows/smoke-test.yml

@@ -0,0 +1,26 @@
+name: Smoke Test
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set Node.js version
+        uses: actions/setup-node@v4
+        with:
+          node-version: '>=22.12.0'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run smoke test against deployed server
+        run: node smokeTest.js

package/apiIntegration.test.js

@@ -0,0 +1,82 @@
+/*
+  apiIntegration.test.js
+  Minimal HTTP integration tests verifying routing, body parsing, and JSON serialization, using the fixture corpus and a temporary in-process server.
+*/
+
+// IMPORTS
+
+const {test, before, after} = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const http = require('node:http');
+const {requestHandler} = require('./index');
+
+// CONSTANTS
+
+const port = 3998;
+
+// SETUP AND TEARDOWN
+
+const savedDBDir = process.env.DB_DIR;
+let server;
+
+before(async () => {
+  process.env.DB_DIR = path.join(__dirname, 'test', 'fixtures', 'db');
+  server = http.createServer(requestHandler);
+  await new Promise(resolve => server.listen(port, resolve));
+});
+
+after(async () => {
+  if (savedDBDir !== undefined) {
+    process.env.DB_DIR = savedDBDir;
+  }
+  else {
+    delete process.env.DB_DIR;
+  }
+  await new Promise(resolve => server.close(() => resolve()));
+});
+
+// HELPERS
+
+// Sends an HTTP request and returns the parsed JSON response body.
+const request = (method, requestPath, body = null) => new Promise((resolve, reject) => {
+  const options = {method, host: 'localhost', port, path: requestPath};
+  if (body) {
+    const bodyJSON = JSON.stringify(body);
+    options.headers = {
+      'content-type': 'application/json; charset=utf-8',
+      'content-length': Buffer.byteLength(bodyJSON)
+    };
+  }
+  const req = http.request(options, response => {
+    const chunks = [];
+    response.on('data', chunk => chunks.push(chunk));
+    response.on('end', () => {
+      const bodyString = chunks.join('');
+      try {
+        resolve({statusCode: response.statusCode, body: JSON.parse(bodyString)});
+      }
+      catch {
+        resolve({statusCode: response.statusCode, body: bodyString});
+      }
+    });
+  });
+  req.on('error', reject);
+  req.end(body ? JSON.stringify(body) : '');
+});
+
+// TESTS
+
+test('GET /api/listReports returns valid JSON through the HTTP routing layer', async () => {
+  const {body} = await request('GET', '/api/listReports');
+  assert.equal(body['tool name'], 'listReports');
+  const reports = body['response content']['basics about all available reports'];
+  assert.equal(reports.length, 6);
+});
+
+test('POST /api/requestFeature parses a JSON body and returns a JSON response through the HTTP routing layer', async () => {
+  const {body} = await request('POST', '/api/requestFeature', {feature: 'Add a dark mode toggle'});
+  const details = body['response content']['details about your request'];
+  assert.equal(details.error, undefined);
+  assert.equal(details.disposition, 'received and logged; manager notified');
+});

package/docs/SERVICE.md

@@ -203,7 +203,7 @@ kilotest.com {
   }
   @allowedPOST {
     method POST
-    path /mcp /requestRetest.html/* /requestTest.html /recAction.html /reannotate.html /renewWCAG.html /api/* /tutorialComment.html
+    path /mcp /requestRetest.html/* /requestTest.html /recAction.html /reannotate.html /renewWCAG.html /api/* /tutorialComment.html /worker/job /worker/report
   }
   # Respond to OPTIONS requests.
   @allowedOPTIONS method OPTIONS
@@ -259,6 +259,30 @@ When a new version of the `kilotest` package has been published, the service can
 1. Update the Playwright browsers: `npx playwright install`
 1. Restart the service: `pm2 restart kilotest`
 
+## Branch protection
+
+The `main` branch on GitHub is protected to prevent direct pushes and to require a passing smoke test before any pull request can merge. The settings are configured in the GitHub repository as a ruleset under Settings > Branches > Branch protection rules.
+
+The required settings are:
+
+1. Require a pull request before merging.
+1. Require status checks to pass before merging.
+1. Add "Smoke Test" as a required status check.
+1. Require branches to be up to date before merging.
+1. Do not allow bypassing the above settings.
+
+A copy of the ruleset is located at `docs/Smoke Test.json`.
+
+The smoke-test check is defined in `.github/workflows/smoke-test.yml` and runs `smokeTest.js`, which sends HTTPS requests to all valid GET and POST paths on the deployed server and verifies that Caddy forwards each one to Kilotest rather than returning a bare 404.
+
+When a pull request adds a new route to `index.js`, the Caddyfile at `/etc/caddy/Caddyfile` on the server must be updated to forward the new path before the pull request can merge. The workflow to follow is:
+
+1. Add the new route to the `routes` constant and the dispatch chain in `index.js`.
+1. Update the Caddyfile (`/etc/caddy/Caddyfile`) on the server to forward the new path.
+1. Open or update the pull request.
+1. The smoke test workflow runs and verifies that all paths, including the new one, are forwarded by Caddy.
+1. If the Caddyfile was not updated, the smoke test reports a bare 404 for the new path, the required status check fails, and the pull request cannot merge.
+
 ## Performance
 
 The Cloud Compute host, in initial testing, took about 2.5 as long to process an example job as an Apple M2 Pro MacBook Pro with 16GB of memory. After tuning, the ratio was reduced to about 1.7.

package/docs/Smoke Test.json

@@ -0,0 +1,54 @@
+{
+  "id": 22391521,
+  "name": "Smoke Test",
+  "target": "branch",
+  "source_type": "Repository",
+  "source": "jrpool/kilotest",
+  "enforcement": "active",
+  "conditions": {
+    "ref_name": {
+      "exclude": [],
+      "include": [
+        "~DEFAULT_BRANCH"
+      ]
+    }
+  },
+  "rules": [
+    {
+      "type": "deletion"
+    },
+    {
+      "type": "non_fast_forward"
+    },
+    {
+      "type": "pull_request",
+      "parameters": {
+        "required_approving_review_count": 0,
+        "dismiss_stale_reviews_on_push": true,
+        "required_reviewers": [],
+        "require_code_owner_review": false,
+        "require_last_push_approval": false,
+        "required_review_thread_resolution": false,
+        "require_extra_approval_for_unattributed_changes": true,
+        "allowed_merge_methods": [
+          "merge",
+          "squash",
+          "rebase"
+        ]
+      }
+    },
+    {
+      "type": "required_status_checks",
+      "parameters": {
+        "strict_required_status_checks_policy": true,
+        "do_not_enforce_on_create": false,
+        "required_status_checks": [
+          {
+            "context": "Smoke Test"
+          }
+        ]
+      }
+    }
+  ],
+  "bypass_actors": []
+}

package/index.js

@@ -63,6 +63,54 @@ const answer = {
 
 // CONSTANTS
 
+// Paths that the application is authorized to handle, by method, as glob-style patterns where * matches any sequence of characters.
+const routes = exports.routes = {
+  GET: [
+    '/mcp',
+    '/',
+    '/index.html',
+    '/robots.txt',
+    '/openapi.yaml',
+    '/openapi.json',
+    '/swagger.yaml',
+    '/swagger.json',
+    '/api-docs',
+    '/llms.txt',
+    '/llms-full.txt',
+    '/sitemap.xml',
+    '/style.css',
+    '/fullReport.json/*',
+    '/api/*',
+    '/tutorial/images/*',
+    '/favicon.*',
+    '*.html*'
+  ],
+  POST: [
+    '/mcp',
+    '/requestTest.html',
+    '/requestRetest.html/*',
+    '/recAction.html',
+    '/reannotate.html',
+    '/renewWCAG.html',
+    '/worker/job',
+    '/worker/report',
+    '/api/*',
+    '/tutorialComment.html'
+  ]
+};
+// Returns whether a pathname matches a glob-style pattern.
+const matchPath = (pattern, pathname) => {
+  const regex = new RegExp(
+    '^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'
+  );
+  return regex.test(pathname);
+};
+// Returns whether a pathname is authorized for a method.
+const isPathAllowed = (method, pathname) => {
+  const patterns = routes[method] || [];
+  return patterns.some(pattern => matchPath(pattern, pathname));
+};
+
 const protocol = process.env.PROTOCOL || 'http';
 const queuePath = () => path.join(jobsPath(), 'queue');
 const claimedPath = () => path.join(jobsPath(), 'claimed');
@@ -321,8 +369,12 @@ const requestHandler = async (request, response) => {
   const pathTail = pathname.split('/').slice(2).join('/');
   // If the request is a GET request:
   if (method === 'GET') {
+    // If the path is not authorized for GET requests:
+    if (!isPathAllowed('GET', pathname)) {
+      await serveError({message: `ERROR: Invalid GET request (${pathname})`}, response, true);
+    }
     // If it is for the model context protocol server:
-    if (pathname === mcpPath) {
+    else if (pathname === mcpPath) {
       // Handle the MCP request.
       await handleMCP(request, response);
     }
@@ -554,8 +606,12 @@ const requestHandler = async (request, response) => {
   }
   // Otherwise, if the request is a POST request:
   else if (method === 'POST') {
+    // If the path is not authorized for POST requests:
+    if (!isPathAllowed('POST', pathname)) {
+      await serveError({message: 'ERROR: Invalid POST request'}, response, true);
+    }
     // If it is for the model context protocol server:
-    if (pageName === 'mcp') {
+    else if (pageName === 'mcp') {
       await handleMCP(request, response);
     }
     // Otherwise, i.e. if it is not for the MCP server:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "46.0.7",
+  "version": "46.0.8",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "engines": {
@@ -10,6 +10,7 @@
     "start": "node index",
     "lint": "npx eslint && npx markdownlint-cli2",
     "test": "c8 node --test",
+    "test:smoke": "node smokeTest",
     "generate:openapi": "node scripts/generate-openapi"
   },
   "repository": {

package/routes.test.js

@@ -0,0 +1,69 @@
+/*
+  routes.test.js
+  Tests for the centralized routes table exported from index.js.
+*/
+
+// IMPORTS
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const {routes} = require('./index');
+
+// TESTS
+
+test('routes has GET and POST arrays', () => {
+  assert.ok(Array.isArray(routes.GET));
+  assert.ok(Array.isArray(routes.POST));
+  assert.ok(routes.GET.length > 0);
+  assert.ok(routes.POST.length > 0);
+});
+
+test('routes GET includes all known GET paths', () => {
+  const required = [
+    '/mcp',
+    '/',
+    '/index.html',
+    '/robots.txt',
+    '/openapi.yaml',
+    '/openapi.json',
+    '/swagger.yaml',
+    '/swagger.json',
+    '/api-docs',
+    '/llms.txt',
+    '/llms-full.txt',
+    '/sitemap.xml',
+    '/style.css'
+  ];
+  for (const path of required) {
+    assert.ok(routes.GET.includes(path), `routes.GET missing ${path}`);
+  }
+});
+
+test('routes POST includes all known POST paths', () => {
+  const required = [
+    '/mcp',
+    '/requestTest.html',
+    '/recAction.html',
+    '/reannotate.html',
+    '/renewWCAG.html',
+    '/worker/job',
+    '/worker/report',
+    '/tutorialComment.html'
+  ];
+  for (const path of required) {
+    assert.ok(routes.POST.includes(path), `routes.POST missing ${path}`);
+  }
+});
+
+test('routes GET includes wildcard patterns for dynamic paths', () => {
+  assert.ok(routes.GET.some(p => p.includes('fullReport.json')));
+  assert.ok(routes.GET.some(p => p.includes('/api/')));
+  assert.ok(routes.GET.some(p => p.includes('tutorial/images')));
+  assert.ok(routes.GET.some(p => p.includes('favicon')));
+  assert.ok(routes.GET.some(p => p.includes('.html')));
+});
+
+test('routes POST includes wildcard patterns for dynamic paths', () => {
+  assert.ok(routes.POST.some(p => p.includes('requestRetest.html')));
+  assert.ok(routes.POST.some(p => p.includes('/api/')));
+});

package/smokeTest.js

@@ -0,0 +1,123 @@
+/*
+  smokeTest.js
+  Sends requests to all valid GET and POST paths on the deployed server and verifies that Caddy forwards them to Kilotest (i.e., the response is not a bare Caddy 404).
+*/
+
+// IMPORTS
+
+const https = require('https');
+const {routes} = require('./index');
+
+// CONSTANTS
+
+const host = process.env.SMOKE_HOST || 'kilotest.com';
+// Concrete paths matching each wildcard pattern in the routes table, for smoke testing.
+const concretePaths = {
+  GET: {
+    '/mcp': '/mcp',
+    '/': '/',
+    '/index.html': '/index.html',
+    '/robots.txt': '/robots.txt',
+    '/openapi.yaml': '/openapi.yaml',
+    '/openapi.json': '/openapi.json',
+    '/swagger.yaml': '/swagger.yaml',
+    '/swagger.json': '/swagger.json',
+    '/api-docs': '/api-docs',
+    '/llms.txt': '/llms.txt',
+    '/llms-full.txt': '/llms-full.txt',
+    '/sitemap.xml': '/sitemap.xml',
+    '/style.css': '/style.css',
+    '/fullReport.json/*': '/fullReport.json/260101T0000/mix',
+    '/api/*': '/api/listReports',
+    '/tutorial/images/*': '/tutorial/images/diagram.png',
+    '/favicon.*': '/favicon.ico',
+    '*.html*': '/listReports.html'
+  },
+  POST: {
+    '/mcp': '/mcp',
+    '/requestTest.html': '/requestTest.html',
+    '/requestRetest.html/*': '/requestRetest.html/260101T0001/ct',
+    '/recAction.html': '/recAction.html',
+    '/reannotate.html': '/reannotate.html',
+    '/renewWCAG.html': '/renewWCAG.html',
+    '/worker/job': '/worker/job',
+    '/worker/report': '/worker/report',
+    '/api/*': '/api/requestFeature',
+    '/tutorialComment.html': '/tutorialComment.html'
+  }
+};
+// Minimal POST bodies for paths that require them.
+const postBodies = {
+  '/api/requestFeature': {feature: 'smoke test'},
+  '/requestTest.html': {what: 'Smoke Test Page', url: 'https://smoketest.example.com', why: 'smoke test'},
+  '/requestRetest.html/260101T0001/ct': {why: 'smoke test'},
+  '/recAction.html': {target: 'https://smoketest.example.com\tSmoke Test Page', authCode: 'invalid'},
+  '/reannotate.html': {authCode: 'invalid'},
+  '/renewWCAG.html': {authCode: 'invalid'},
+  '/worker/job': {},
+  '/worker/report': {},
+  '/tutorialComment.html': {content: 'smoke test'},
+  '/mcp': {}
+};
+
+// FUNCTIONS
+
+// Sends an HTTPS request and returns the status code and body length.
+const sendRequest = (method, requestPath) => new Promise((resolve, reject) => {
+  const body = method === 'POST' ? JSON.stringify(postBodies[requestPath] || {}) : null;
+  const options = {
+    method,
+    host,
+    path: requestPath,
+    headers: body ? {
+      'content-type': 'application/json; charset=utf-8',
+      'content-length': Buffer.byteLength(body)
+    } : {}
+  };
+  const req = https.request(options, response => {
+    const chunks = [];
+    response.on('data', chunk => chunks.push(chunk));
+    response.on('end', () => {
+      const responseBody = chunks.join('');
+      resolve({statusCode: response.statusCode, bodyLength: responseBody.length});
+    });
+  });
+  req.on('error', reject);
+  req.end(body || '');
+});
+
+// Returns whether a response is a bare Caddy 404 (the failure this test detects).
+const isCaddy404 = result => result.statusCode === 404 && result.bodyLength === 0;
+
+// EXECUTION
+
+(async () => {
+  let failures = 0;
+  for (const method of ['GET', 'POST']) {
+    console.log(`\n=== ${method} paths ===`);
+    for (const pattern of routes[method]) {
+      const requestPath = concretePaths[method][pattern];
+      if (!requestPath) {
+        console.log(`FAIL: no concrete path defined for pattern ${pattern}`);
+        failures++;
+        continue;
+      }
+      try {
+        const result = await sendRequest(method, requestPath);
+        if (isCaddy404(result)) {
+          console.log(`FAIL: ${method} ${requestPath} -> Caddy 404 (path not forwarded)`);
+          failures++;
+        }
+        else {
+          console.log(`PASS: ${method} ${requestPath} -> ${result.statusCode} (body ${result.bodyLength} bytes)`);
+        }
+      }
+      catch (error) {
+        console.log(`FAIL: ${method} ${requestPath} -> error: ${error.message}`);
+        failures++;
+      }
+    }
+  }
+  console.log(`\n=== ${failures === 0 ? 'All paths forwarded' : `${failures} failure(s)`} ===`);
+  process.exit(failures === 0 ? 0 : 1);
+})();