@jrpool/kilotest 46.0.8 → 46.0.11

package/indexBalances.test.js

@@ -0,0 +1,188 @@
+/*
+  indexBalances.test.js
+  Tests for checkBalancesForAlerts in index.js, which requires balance-related
+  env vars to be set before index.js is loaded.
+*/
+
+// ENVIRONMENT (must be set before requiring index.js)
+
+const path = require('node:path');
+const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
+
+process.env.DB_DIR = fixtureDBDir;
+process.env.AUTH_CODE = 'test-auth-code';
+process.env.TESTARO_WORKERS = JSON.stringify({
+  worker1: {secret: 'secret1', name: 'Worker One'}
+});
+process.env.WAVE_BALANCE_THRESHOLD = '100';
+process.env.AI_SERVICE0_BALANCE_THRESHOLD = '50';
+process.env.AI_MODEL0_INPUT_PRICE = '0.001';
+process.env.AI_MODEL0_OUTPUT_PRICE = '0.002';
+
+const {test, before, after} = require('node:test');
+const assert = require('node:assert/strict');
+const http = require('node:http');
+const fs = require('node:fs/promises');
+const {requestHandler} = require('./index');
+
+// CONSTANTS
+
+const port = 3994;
+const balancePath = path.join(__dirname, 'ai0Balance.json');
+const jobID = '990101T0001-bal';
+const reportPath = path.join(fixtureDBDir, 'reports', `${jobID}.json`);
+const claimedDir = path.join(fixtureDBDir, 'jobs', 'claimed');
+const jobPath = path.join(claimedDir, `${jobID}.json`);
+
+// SETUP AND TEARDOWN
+
+let server;
+let originalBalance;
+
+before(async () => {
+  originalBalance = await fs.readFile(balancePath, 'utf8').catch(() => null);
+  server = http.createServer(requestHandler);
+  await new Promise(resolve => server.listen(port, () => resolve()));
+});
+
+after(async () => {
+  server.closeAllConnections?.();
+  await new Promise(resolve => {
+    const timer = setTimeout(() => {
+      server.closeAllConnections?.();
+      resolve();
+    }, 1000);
+    server.close(() => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+  if (originalBalance !== null) {
+    await fs.writeFile(balancePath, originalBalance);
+  }
+  else {
+    await fs.unlink(balancePath).catch(() => {});
+  }
+  // Clean up any report and job files created by tests.
+  await fs.unlink(reportPath).catch(() => {});
+  await fs.unlink(jobPath).catch(() => {});
+});
+
+// HELPERS
+
+const request = (method, requestPath, body = null, headers = {}) => new Promise((resolve, reject) => {
+  const options = {method, host: 'localhost', port, path: requestPath, headers: {...headers}};
+  let bodyData = '';
+  if (body) {
+    bodyData = JSON.stringify(body);
+    options.headers['content-type'] = 'application/json';
+    options.headers['content-length'] = Buffer.byteLength(bodyData);
+  }
+  const req = http.request(options, response => {
+    const chunks = [];
+    response.on('data', chunk => chunks.push(chunk));
+    response.on('end', () => {
+      resolve({statusCode: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString()});
+    });
+  });
+  req.on('error', reject);
+  req.end(bodyData || '');
+});
+
+const jsonBody = res => {
+  try {
+    return JSON.parse(res.body);
+  }
+  catch {
+    return null;
+  }
+};
+
+const submitReport = async (acts, balanceFileContent) => {
+  if (balanceFileContent !== undefined) {
+    if (balanceFileContent === null) {
+      await fs.unlink(balancePath).catch(() => {});
+    }
+    else {
+      await fs.writeFile(balancePath, balanceFileContent);
+    }
+  }
+  await fs.writeFile(jobPath, JSON.stringify({
+    id: jobID,
+    target: {what: 'Test', url: 'https://example.com/test'},
+    sources: {worker: 'Worker One'}
+  }));
+  const auth = Buffer.from('worker1:secret1').toString('base64');
+  const report = {
+    id: jobID,
+    target: {what: 'Test', url: 'https://example.com/test'},
+    acts,
+    jobData: {endTime: '26-01-01T00:00'},
+    catalog: {}
+  };
+  const res = await request('POST', '/worker/report', {report}, {
+    authorization: `Basic ${auth}`
+  });
+  await fs.unlink(jobPath).catch(() => {});
+  await fs.unlink(reportPath).catch(() => {});
+  return res;
+};
+
+// TESTS
+
+test('worker/report with WAVE balance low triggers a balance alert', async () => {
+  const res = await submitReport(
+    [
+      {type: 'test', which: 'wave', data: {creditsRemaining: 5}},
+      {type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
+    ],
+    JSON.stringify({balance: 100})
+  );
+  assert.equal(res.statusCode, 200);
+  const body = jsonBody(res);
+  assert.equal(body.status, 'ok');
+});
+
+test('worker/report with AI service balance low triggers a balance alert', async () => {
+  const res = await submitReport(
+    [
+      {type: 'test', which: 'wave', data: {creditsRemaining: 200}},
+      {type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 5000, outputTokens: 3000}}}}}
+    ],
+    JSON.stringify({balance: 10})
+  );
+  assert.equal(res.statusCode, 200);
+});
+
+test('worker/report with no balance file logs an error', async () => {
+  const res = await submitReport(
+    [
+      {type: 'test', which: 'wave', data: {creditsRemaining: 200}},
+      {type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
+    ],
+    null
+  );
+  assert.equal(res.statusCode, 200);
+});
+
+test('worker/report with invalid balance JSON logs an error', async () => {
+  const res = await submitReport(
+    [
+      {type: 'test', which: 'wave', data: {creditsRemaining: 200}},
+      {type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
+    ],
+    'not valid json'
+  );
+  assert.equal(res.statusCode, 200);
+});
+
+test('worker/report with non-number balance logs an error', async () => {
+  const res = await submitReport(
+    [
+      {type: 'test', which: 'wave', data: {creditsRemaining: 200}},
+      {type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
+    ],
+    JSON.stringify({balance: 'not a number'})
+  );
+  assert.equal(res.statusCode, 200);
+});

package/indexFallback.test.js

@@ -0,0 +1,33 @@
+/*
+  indexFallback.test.js
+  Tests that index.js loads successfully when PROTOCOL and TESTARO_WORKERS
+  are not set, exercising the || fallback branches at module load time.
+  This must be a separate file because index.js reads these env vars
+  at module load time, and each test file runs in its own process.
+*/
+
+// ENVIRONMENT (must be set before requiring index.js)
+
+const path = require('node:path');
+const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
+
+process.env.DB_DIR = fixtureDBDir;
+process.env.AUTH_CODE = 'test-auth-code';
+// Set PROTOCOL and TESTARO_WORKERS to empty strings (falsy) so that
+// the || fallbacks in index.js are exercised. dotenv does not override
+// existing env vars, so .env will not replace these.
+process.env.PROTOCOL = '';
+process.env.TESTARO_WORKERS = '';
+
+// IMPORTS
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const {requestHandler} = require('./index');
+
+// TESTS
+
+test('index.js loads with empty PROTOCOL and TESTARO_WORKERS using fallbacks', () => {
+  // If index.js loaded successfully, requestHandler is a function.
+  assert.equal(typeof requestHandler, 'function');
+});

package/indexServe.test.js

@@ -0,0 +1,59 @@
+/*
+  indexServe.test.js
+  Tests for the serve function in index.js, which starts the HTTP or HTTPS
+  server. The serve function is exported for testing.
+*/
+
+// ENVIRONMENT (must be set before requiring index.js)
+
+const path = require('node:path');
+const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
+
+process.env.DB_DIR = fixtureDBDir;
+process.env.AUTH_CODE = 'test-auth-code';
+process.env.TESTARO_WORKERS = '{}';
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const http = require('node:http');
+const fs = require('node:fs/promises');
+const {serve} = require('./index');
+
+// HELPER
+
+const closeServer = server => new Promise(resolve => {
+  if (!server) {
+    resolve();
+    return;
+  }
+  // Close all idle connections, then close the server.
+  server.closeAllConnections?.();
+  const timer = setTimeout(() => {
+    server.closeAllConnections?.();
+    resolve();
+  }, 500);
+  server.close(() => {
+    clearTimeout(timer);
+    resolve();
+  });
+});
+
+// TESTS
+
+test('serve creates missing directories and returns an HTTP server', async () => {
+  // Remove the queue directory to verify serve recreates it.
+  const queueDir = path.join(fixtureDBDir, 'jobs', 'queue');
+  await fs.rm(queueDir, {recursive: true}).catch(() => {});
+  process.env.PORT = '3987';
+  const server = await serve(http, {});
+  try {
+    assert.ok(server);
+    assert.equal(typeof server.listen, 'function');
+    // Verify the queue directory was recreated.
+    const exists = await fs.access(queueDir).then(() => true).catch(() => false);
+    assert.ok(exists, 'serve should create the queue directory');
+  }
+  finally {
+    await closeServer(server);
+  }
+});

package/indexWorkers.test.js

@@ -0,0 +1,79 @@
+/*
+  indexWorkers.test.js
+  Tests for index.js when TESTARO_WORKERS is invalid JSON.
+  This must be a separate file because index.js reads TESTARO_WORKERS
+  at module load time.
+*/
+
+// ENVIRONMENT (must be set before requiring index.js)
+
+const path = require('node:path');
+const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
+
+process.env.DB_DIR = fixtureDBDir;
+process.env.AUTH_CODE = 'test-auth-code';
+process.env.TESTARO_WORKERS = 'not valid json';
+
+// IMPORTS
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const http = require('node:http');
+const {requestHandler} = require('./index');
+
+// CONSTANTS
+
+const port = 3993;
+
+// HELPERS
+
+const request = (method, requestPath, body = null, headers = {}) => new Promise((resolve, reject) => {
+  const options = {method, host: 'localhost', port, path: requestPath, headers: {...headers}};
+  let bodyData = '';
+  if (body) {
+    bodyData = JSON.stringify(body);
+    options.headers['content-type'] = 'application/json';
+    options.headers['content-length'] = Buffer.byteLength(bodyData);
+  }
+  const req = http.request(options, response => {
+    const chunks = [];
+    response.on('data', chunk => chunks.push(chunk));
+    response.on('end', () => {
+      resolve({statusCode: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString()});
+    });
+  });
+  req.on('error', reject);
+  req.end(bodyData || '');
+});
+
+// TESTS
+
+test('index.js loads with invalid TESTARO_WORKERS and treats workers as empty', () => {
+  // If index.js loaded successfully, requestHandler is a function.
+  assert.equal(typeof requestHandler, 'function');
+});
+
+test('POST /worker/job with invalid TESTARO_WORKERS returns 401 for any credentials', async () => {
+  const server = http.createServer(requestHandler);
+  await new Promise(resolve => server.listen(port, () => resolve()));
+  try {
+    const auth = Buffer.from('worker1:secret1').toString('base64');
+    const res = await request('POST', '/worker/job', {}, {
+      authorization: `Basic ${auth}`
+    });
+    assert.equal(res.statusCode, 401);
+  }
+  finally {
+    server.closeAllConnections?.();
+    await new Promise(resolve => {
+      const timer = setTimeout(() => {
+        server.closeAllConnections?.();
+        resolve();
+      }, 1000);
+      server.close(() => {
+        clearTimeout(timer);
+        resolve();
+      });
+    });
+  }
+});

package/mcp.js

@@ -44,7 +44,7 @@ exports.mcpPath = '/mcp';
 // FUNCTIONS
 
 // Creates and returns an McpServer with Kilotest tools registered.
-const createMCPServer = () => {
+exports.createMCPServer = () => {
   const server = new McpServer({
     name: 'Kilotest',
     version,
@@ -210,7 +210,7 @@ const createMCPServer = () => {
 // Handles an MCP request.
 exports.handleMCP = async (request, response) => {
   const transport = new StreamableHTTPServerTransport({sessionIdGenerator: undefined});
-  const server = createMCPServer();
+  const server = exports.createMCPServer();
   await server.connect(transport);
   await transport.handleRequest(request, response);
 };

package/mcp.test.js

@@ -0,0 +1,281 @@
+/*
+  mcp.test.js
+  Tests for mcp.js, covering tool registration and handler behavior.
+*/
+
+// IMPORTS
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const http = require('node:http');
+const {mcpPath, createMCPServer} = require('./mcp');
+
+// CONSTANTS
+
+// Set DB_DIR to the fixture database for all tests.
+process.env.DB_DIR = path.join(__dirname, 'test', 'fixtures', 'db');
+
+// TESTS
+
+test('mcpPath is /mcp', () => {
+  assert.equal(mcpPath, '/mcp');
+});
+
+test('createMCPServer registers all 8 tools', () => {
+  const server = createMCPServer();
+  const toolNames = Object.keys(server._registeredTools);
+  assert.equal(toolNames.length, 8);
+  assert.deepEqual(toolNames, [
+    'listReports',
+    'listIssues',
+    'listViolators',
+    'listDiagnoses',
+    'getReport',
+    'requestTest',
+    'requestRetest',
+    'requestFeature'
+  ]);
+});
+
+test('each tool has a description and a handler function', () => {
+  const server = createMCPServer();
+  const tools = server._registeredTools;
+  for (const [name, tool] of Object.entries(tools)) {
+    assert.ok(tool.description, `${name} has a description`);
+    assert.equal(typeof tool.handler, 'function', `${name} has a handler function`);
+  }
+});
+
+test('listReports handler returns content and structuredContent', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listReports.handler({});
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('listIssues handler returns content and structuredContent for a valid report', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listIssues.handler({
+    timeStamp: '260101T0000',
+    jobID: 'mix'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('listViolators handler returns content and structuredContent for a valid issue', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listViolators.handler({
+    issueID: 'linkNoText',
+    timeStamp: '260101T0000',
+    jobID: 'mix'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('listDiagnoses handler returns content and structuredContent for a valid diagnosis', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listDiagnoses.handler({
+    catalogIndex: '0',
+    issueID: 'linkNoText',
+    timeStamp: '260101T0000',
+    jobID: 'mix'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('getReport handler returns content and structuredContent for a valid report', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.getReport.handler({
+    timeStamp: '260101T0000',
+    jobID: 'mix'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('requestTest handler returns content and structuredContent', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.requestTest.handler({
+    description: 'Test Page',
+    URL: 'https://example.com/test',
+    reason: 'Because accessibility matters'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('requestRetest handler returns content and structuredContent for a valid report', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.requestRetest.handler({
+    timeStamp: '260101T0000',
+    jobID: 'mix',
+    reason: 'Because the report is obsolete'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('requestFeature handler returns content and structuredContent', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.requestFeature.handler({
+    feature: 'A new feature idea'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+});
+
+test('listIssues handler returns an error for a nonexistent report', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listIssues.handler({
+    timeStamp: '990101T0000',
+    jobID: 'xxx'
+  });
+  const reportBasics = result.structuredContent['response content']['basics about the report'];
+  assert.ok(reportBasics.error);
+});
+
+test('listViolators handler returns an error for an unknown issue', async () => {
+  const server = createMCPServer();
+  const result = await server._registeredTools.listViolators.handler({
+    issueID: 'nonexistentIssue',
+    timeStamp: '260101T0000',
+    jobID: 'mix'
+  });
+  const issueBasics = result.structuredContent['response content']['basics about the issue'];
+  assert.ok(issueBasics.error);
+});
+
+// INTEGRATION TESTS FOR handleMCP
+
+// Helper: sends a single MCP JSON-RPC request to a local server and returns the parsed SSE response.
+const sendMCPRequest = (port, method, params, id) => new Promise((resolve, reject) => {
+  const body = JSON.stringify({jsonrpc: '2.0', method, params, id});
+  const req = http.request({
+    port,
+    method: 'POST',
+    path: '/mcp',
+    headers: {
+      'content-type': 'application/json',
+      'accept': 'application/json, text/event-stream',
+      'content-length': Buffer.byteLength(body)
+    }
+  }, res => {
+    let data = '';
+    res.on('data', chunk => {
+      data += chunk;
+    });
+    res.on('end', () => {
+      resolve({statusCode: res.statusCode, headers: res.headers, body: data});
+    });
+  });
+  req.on('error', reject);
+  req.write(body);
+  req.end();
+});
+
+// Helper: parses the JSON-RPC result from an SSE response body.
+const parseSSEResult = body => {
+  const jsonLine = body.split('\n').find(line => line.startsWith('data: '));
+  if (!jsonLine) {
+    throw new Error('No data line in SSE response');
+  }
+  return JSON.parse(jsonLine.slice(6));
+};
+
+// Helper: starts a local HTTP server with handleMCP and returns it.
+const startMCPServer = () => new Promise(resolve => {
+  const {handleMCP} = require('./mcp');
+  const server = http.createServer((req, res) => handleMCP(req, res));
+  server.listen(0, () => resolve(server));
+});
+
+// Helper: closes a server with a timeout fallback.
+const closeMCPServer = server => new Promise(resolve => {
+  if (!server) {
+    resolve();
+    return;
+  }
+  server.closeAllConnections?.();
+  const timer = setTimeout(() => {
+    server.closeAllConnections?.();
+    resolve();
+  }, 1000);
+  server.close(() => {
+    clearTimeout(timer);
+    resolve();
+  });
+});
+
+test('handleMCP responds to initialize with server info', async () => {
+  const server = await startMCPServer();
+  try {
+    const port = server.address().port;
+    const res = await sendMCPRequest(port, 'initialize', {
+      protocolVersion: '2025-06-18',
+      capabilities: {},
+      clientInfo: {name: 'test', version: '1.0'}
+    }, 1);
+    assert.equal(res.statusCode, 200);
+    const result = parseSSEResult(res.body);
+    assert.equal(result.result.serverInfo.name, 'Kilotest');
+    assert.equal(result.result.protocolVersion, '2025-06-18');
+  }
+  finally {
+    await closeMCPServer(server);
+  }
+});
+
+test('handleMCP lists all 8 tools via tools/list', async () => {
+  const server = await startMCPServer();
+  try {
+    const port = server.address().port;
+    const res = await sendMCPRequest(port, 'tools/list', {}, 2);
+    assert.equal(res.statusCode, 200);
+    const result = parseSSEResult(res.body);
+    const toolNames = result.result.tools.map(t => t.name);
+    assert.equal(toolNames.length, 8);
+    assert.deepEqual(toolNames, [
+      'listReports',
+      'listIssues',
+      'listViolators',
+      'listDiagnoses',
+      'getReport',
+      'requestTest',
+      'requestRetest',
+      'requestFeature'
+    ]);
+  }
+  finally {
+    await closeMCPServer(server);
+  }
+});
+
+test('handleMCP executes listReports tool via tools/call', async () => {
+  const server = await startMCPServer();
+  try {
+    const port = server.address().port;
+    const res = await sendMCPRequest(port, 'tools/call', {
+      name: 'listReports',
+      arguments: {}
+    }, 3);
+    assert.equal(res.statusCode, 200);
+    const result = parseSSEResult(res.body);
+    assert.ok(result.result.content);
+    assert.equal(result.result.content[0].type, 'text');
+  }
+  finally {
+    await closeMCPServer(server);
+  }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "46.0.8",
+  "version": "46.0.11",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "engines": {
@@ -9,7 +9,7 @@
   "scripts": {
     "start": "node index",
     "lint": "npx eslint && npx markdownlint-cli2",
-    "test": "c8 node --test",
+    "test": "c8 node --require ./test/setup.cjs --test --test-concurrency=1 --test-timeout=70",
     "test:smoke": "node smokeTest",
     "generate:openapi": "node scripts/generate-openapi"
   },
@@ -66,7 +66,8 @@
       "mcp.js",
       "alerts.js",
       "util.js",
-      "api/**/*.js"
+      "api/**/*.js",
+      "web/**/*.js"
     ],
     "exclude": [
       "coverage/**",
@@ -74,8 +75,12 @@
       "test{,-*}.{js,cjs,mjs}",
       "**/*{.,-}test.{js,cjs,mjs}",
       "**/__tests__/**",
-      "web/**",
       "scripts/**"
-    ]
+    ],
+    "check-coverage": true,
+    "lines": 100,
+    "statements": 100,
+    "functions": 100,
+    "branches": 100
   }
 }