@jrpool/kilotest 61.0.0 → 63.0.0

package/.devin/config.local.json

@@ -41,7 +41,8 @@
       "Exec(npm update)",
       "Exec(npm ls)",
       "Exec(pkill)",
-      "Exec(python3)"
+      "Exec(python3)",
+      "Exec(npx)"
     ]
   }
 }

package/.markdownlint-cli2.jsonc

@@ -3,7 +3,6 @@
     "**/*.md"
   ],
   "ignores": [
-    "docs/IDEAS.md",
     "node_modules/**"
   ],
   "config": {

package/api/awaitTest.test.ts

@@ -29,6 +29,7 @@ before(() => {
 });
 
 import {response} from './awaitTest.ts';
+import {awaitTestResponseSchema} from './schemas.ts';
 
 after(() => {
   delete process.env.AWAIT_JOB_POLL_MS;
@@ -112,3 +113,11 @@ test('awaitTest reports both orderNewTest and orderRetest as possible ancestors'
     assert.equal(ancestor.method, 'POST');
   }
 });
+
+test('awaitTest response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['999999T9999-xyz']);
+  assert.deepEqual(awaitTestResponseSchema.parse(body), body);
+});

package/api/getReport.test.ts

@@ -8,6 +8,7 @@
 import {test, before, after} from 'node:test';
 import assert from 'node:assert/strict';
 import {response} from './getReport.ts';
+import {getReportResponseSchema} from './schemas.ts';
 
 // SETUP AND TEARDOWN
 
@@ -59,3 +60,11 @@ test('getReport includes tool name and metadata', async () => {
   assert.equal(body['tool name'], 'getReport');
   assert.ok(body['response metadata'].identifier);
 });
+
+test('getReport response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['260101T0000-mix']);
+  assert.deepEqual(getReportResponseSchema.parse(body), body);
+});

package/api/listDiagnoses.test.ts

@@ -8,6 +8,7 @@
 import {test, before, after} from 'node:test';
 import assert from 'node:assert/strict';
 import {response} from './listDiagnoses.ts';
+import {listDiagnosesResponseSchema} from './schemas.ts';
 
 // SETUP AND TEARDOWN
 
@@ -129,3 +130,11 @@ test('listDiagnoses returns issue error for a falsy issue ID', async () => {
   const issueBasics = body['response content']['basics about the issue'] as any;
   assert.ok(issueBasics.error);
 });
+
+test('listDiagnoses response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['0', 'linkNoText', '260101T0000-mix']);
+  assert.deepEqual(listDiagnosesResponseSchema.parse(body), body);
+});

package/api/listDiagnoses.ts

@@ -94,14 +94,24 @@ export const response = async (args: string[]) => {
         const diagnoses: Diagnoses = [];
         // For each instance reporting a violation of the issue by the violator:
         getTestActInstances(report, {violationsOnly: true, issueID, catalogIndex})
-        .forEach(({instance}) => {
+        .forEach(({act, instance}) => {
           const {count, ordinalSeverity, ruleID, what} = instance;
           // Get the diagnosis.
-          const diagnosis = {
+          const diagnosis: Diagnoses[number] = {
+            'identifier of the rule engine': act.which!,
             'identifier of the violated rule': ruleID !== what ? ruleID : null,
             'description of the violation': what,
             'severity of the violation on a 0-to-3 scale': ordinalSeverity,
-            'count of violations of the rule by the element': count ?? 1
+            'count of violations of the rule by the element': count ?? 1,
+            'how to report that you disagree with this diagnosis': {
+              method: 'POST',
+              URL: `${thisHost}/api/reportDisagreement/${issueID}/${reportID}/${catalogIndex}`,
+              'request body': {
+                engineID: 'the identifier of the rule engine shown in this diagnosis',
+                ruleID: 'the identifier of the violated rule shown in this diagnosis, or, if it is null, the description of the violation',
+                reason: '20- to 1000-character explanation of why you disagree with this diagnosis'
+              }
+            }
           }
           diagnoses.push(diagnosis);
         });

package/api/listIssues.test.ts

@@ -8,6 +8,7 @@
 import {test, before, after} from 'node:test';
 import assert from 'node:assert/strict';
 import {response} from './listIssues.ts';
+import {listIssuesResponseSchema} from './schemas.ts';
 
 // SETUP AND TEARDOWN
 
@@ -145,3 +146,11 @@ test('listIssues gives no history for a nonexistent report', async () => {
   const body = await response(['999999T9999-xyz']);
   assert.equal(body['response content']['history of reports about the page'], null);
 });
+
+test('listIssues response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['260101T0000-mix']);
+  assert.deepEqual(listIssuesResponseSchema.parse(body), body);
+});

package/api/listIssues.ts

@@ -31,6 +31,10 @@ import {listIssuesResponseSchema} from './schemas.ts';
 
 // The response content defined by the response schema.
 type ResponseContent = z.infer<typeof listIssuesResponseSchema>['response content'];
+// One entry of the history of reports about the page.
+type ReportHistoryEntry = NonNullable<ResponseContent['history of reports about the page']>[number];
+// The basics about one issue in the report.
+type IssueBasics = NonNullable<ResponseContent['basics about all issues reported in the report']>[number];
 
 // FUNCTIONS
 
@@ -67,7 +71,7 @@ export const response = async (args: string[]) => {
       const pageExtracts = await getPageReportExtracts(report.target.what);
       const lastIndex = pageExtracts.length - 1;
       // Add the history of reports about the page to the response content.
-      responseContent['history of reports about the page'] = pageExtracts.map((extract, index) => {
+      responseContent['history of reports about the page'] = pageExtracts.map((extract, index): ReportHistoryEntry => {
         const extractID = extract.id;
         return {
           identifier: extractID,
@@ -77,7 +81,7 @@ export const response = async (args: string[]) => {
           'whether it is the latest report about the page': index === lastIndex,
           'whether it is the report described in this response': extractID === reportID,
           'how to get details about the report': {
-            method: 'GET' as const,
+            method: 'GET',
             URL: `${thisHost}/api/listIssues/${extractID}`
           },
           'web users can get details about the report at': `${thisHost}/listIssues.html/${extractID}`
@@ -210,7 +214,7 @@ export const response = async (args: string[]) => {
       // Sort the data about issues by summary.
       const sortedIssuesData = objectSort(Object.values(issuesData), 'summary', 'alpha');
       // Get the basics about the issues.
-      const issuesBasics = sortedIssuesData.map(issueData => {
+      const issuesBasics = sortedIssuesData.map((issueData): IssueBasics => {
         const {id, summary, weight, why, reporterIDs} = issueData;
         return {
           identifier: id,
@@ -220,7 +224,7 @@ export const response = async (args: string[]) => {
           'rule engines with any violations belonging to the issue': getRuleEnginesFacts(reporterIDs)
           .map(ruleEnginesFact => ruleEnginesFact.name),
           'how to get details about the issue': {
-            method: 'GET' as const,
+            method: 'GET',
             URL: `${thisHost}/api/listViolators/${id}/${reportID}`
           },
           'web users can get details about the issue at': `${thisHost}/listViolators.html/${id}/${reportID}`

package/api/listPages.test.ts

@@ -8,6 +8,7 @@
 import {test, before, after} from 'node:test';
 import assert from 'node:assert/strict';
 import {response} from './listPages.ts';
+import {listPagesResponseSchema} from './schemas.ts';
 
 // SETUP AND TEARDOWN
 
@@ -91,3 +92,11 @@ test('listPages includes tool name and metadata', async () => {
   assert.ok(body['response metadata'].identifier);
   assert.ok(body['response metadata']['date and time']);
 });
+
+test('listPages response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response();
+  assert.deepEqual(listPagesResponseSchema.parse(body), body);
+});

package/api/listViolators.test.ts

@@ -8,6 +8,7 @@
 import {test, before, after} from 'node:test';
 import assert from 'node:assert/strict';
 import {response} from './listViolators.ts';
+import {listViolatorsResponseSchema} from './schemas.ts';
 
 // SETUP AND TEARDOWN
 
@@ -105,3 +106,11 @@ test('listViolators returns an error for a falsy issue ID', async () => {
   const issueBasics = body['response content']['basics about the issue'] as any;
   assert.ok(issueBasics.error);
 });
+
+test('listViolators response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['linkNoText', '260101T0000-mix']);
+  assert.deepEqual(listViolatorsResponseSchema.parse(body), body);
+});

package/api/listViolators.ts

@@ -21,6 +21,8 @@ import {listViolatorsResponseSchema} from './schemas.ts';
 
 // The response content defined by the response schema.
 type ResponseContent = z.infer<typeof listViolatorsResponseSchema>['response content'];
+// The basics about one element exhibiting the issue.
+type ViolatorBasics = NonNullable<ResponseContent['basics about all elements exhibiting the issue']>[number];
 
 // FUNCTIONS
 
@@ -96,7 +98,7 @@ export const response = async (args: string[]) => {
       );
       // Get basics about the violators.
       const violatorsBasics = sortedViolatorData
-      .map(violator => {
+      .map((violator): ViolatorBasics => {
         const {catalogIndex, reporters} = violator;
         const catalogItem = catalog[catalogIndex];
         return {
@@ -105,8 +107,8 @@ export const response = async (args: string[]) => {
           'inner text': catalogItem?.text ?? null,
           'count of rule engines reporting that the element exhibited the issue': reporters.size,
           'how to get details about the element': {
-            URL: `${thisHost}/api/listDiagnoses/${catalogIndex}/${issueID}/${reportID}`,
-            'request method': 'GET' as const
+            method: 'GET',
+            URL: `${thisHost}/api/listDiagnoses/${catalogIndex}/${issueID}/${reportID}`
           },
           'web users can get details about the element at': `${thisHost}/listDiagnoses.html/${issueID}/${reportID}/${catalogIndex}`
         };

package/api/orderNewTest.test.ts

@@ -45,6 +45,7 @@ beforeEach(async () => {
 });
 
 import {response} from './orderNewTest.ts';
+import {orderNewTestResponseSchema} from './schemas.ts';
 
 after(() => {
   console.log = originalLog;
@@ -210,3 +211,11 @@ test('orderNewTest response omits a web UI URL for this request but includes the
   assert.equal(similarWeb['this request'], null);
   assert.ok(similarWeb['closest ancestor request'].includes('listPages.html'));
 });
+
+test('orderNewTest response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['Schema Conformance Ordered Page', 'https://example.com/schema-ordered', 'A reason that is long enough.']);
+  assert.deepEqual(orderNewTestResponseSchema.parse(body), body);
+});

package/api/orderRetest.test.ts

@@ -48,6 +48,7 @@ beforeEach(async () => {
 });
 
 import {response} from './orderRetest.ts';
+import {orderRetestResponseSchema} from './schemas.ts';
 
 after(async () => {
   await fs.unlink(retestFixturePath);
@@ -177,3 +178,11 @@ test('orderRetest response omits a web UI URL for this request but includes the
   assert.equal(similarWeb['this request'], null);
   assert.ok(similarWeb['closest ancestor request'].includes('listIssues.html'));
 });
+
+test('orderRetest response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['260101T0004-ord', 'A reason that is long enough.']);
+  assert.deepEqual(orderRetestResponseSchema.parse(body), body);
+});

package/api/reportDisagreement.test.ts

@@ -0,0 +1,217 @@
+/*
+  reportDisagreement.test.ts
+  Tests for api/reportDisagreement.ts using the fixture corpus, with mocked side effects.
+*/
+
+// IMPORTS
+
+import {test, before, beforeEach, after} from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+// Blank the alert configuration unconditionally, before ./reportDisagreement.ts (which
+// calls sendAlert) is ever imported below, so this file sends no real alert emails
+// even when run directly rather than via `npm test`.
+for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) {
+  process.env[key] = '';
+}
+
+// ENVIRONMENT
+
+const testDir = path.join(import.meta.dirname, '../test/fixtures/disagreements');
+process.env.DISAGREEMENTS_PATH = path.join(testDir, 'disagreements.json');
+
+// SETUP AND TEARDOWN
+
+const savedDBDir = process.env.DB_DIR;
+let logged: any[] = [];
+const originalLog = console.log;
+
+before(async () => {
+  process.env.DB_DIR = (await import('../test/dbFixture.ts')).fixtureDBDir;
+});
+
+// Capture console.log so the alert sendAlert emits can be observed. The alert
+// configuration is blanked above, so sendAlert logs a WARNING instead of sending.
+beforeEach(async () => {
+  logged = [];
+  console.log = (...args) => logged.push(args.join(' '));
+  await fs.mkdir(testDir, {recursive: true});
+  await fs.writeFile(process.env.DISAGREEMENTS_PATH!, '[]\n');
+});
+
+import {response} from './reportDisagreement.ts';
+import {reportDisagreementResponseSchema} from './schemas.ts';
+
+after(async () => {
+  console.log = originalLog;
+  if (savedDBDir !== undefined) {
+    process.env.DB_DIR = savedDBDir;
+  }
+  else {
+    delete process.env.DB_DIR;
+  }
+  await fs.rm(testDir, {recursive: true, force: true});
+});
+
+// TESTS
+
+test('reportDisagreement rejects a reason shorter than 20 characters', async () => {
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'Too short']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('reason'));
+  assert.ok(
+    !logged.some(line => line.startsWith('WARNING (Kilotest: diagnosis disagreement received)'))
+  );
+});
+
+test('reportDisagreement rejects a reason longer than 1000 characters', async () => {
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'x'.repeat(1001)]);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('reason'));
+});
+
+test('reportDisagreement rejects a nonexistent report', async () => {
+  const body = await response(['linkNoText', '999999T9999-xyz', '0', 'axe', 'r11', 'This diagnosis is wrong, in my opinion']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('report'));
+});
+
+test('reportDisagreement rejects an element not in the report', async () => {
+  const body = await response(['linkNoText', '260101T0000-mix', '999', 'axe', 'r11', 'This diagnosis is wrong, in my opinion']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('element'));
+});
+
+test('reportDisagreement rejects a diagnosis no rule engine reported', async () => {
+  // The mixed report has axe and alfa diagnoses of rule r11 for element 0 and issue
+  // linkNoText, but no ibm diagnosis, so this engineID/ruleID pair matches nothing.
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'ibm', 'r11', 'This diagnosis is wrong, in my opinion']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('no such diagnosis'));
+});
+
+test('reportDisagreement rejects a rule ID reported only under a different issue', async () => {
+  const body = await response(['focusIndicationBad', '260101T0000-mix', '0', 'axe', 'r11', 'This diagnosis is wrong, in my opinion']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('no such diagnosis'));
+});
+
+test('reportDisagreement accepts a valid disagreement and notifies the manager', async () => {
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+  const details = body['response content']['details about your request'] as any;
+  assert.equal(details.error, undefined);
+  assert.ok(details['date and time received']);
+  assert.equal(details['diagnosis you disagree with']['identifier of the rule engine'], 'axe');
+  assert.equal(details['diagnosis you disagree with']['identifier of the violated rule'], 'r11');
+  assert.equal(details.disposition, 'received and logged; manager notified');
+  assert.ok(logged.some(line =>
+    line.startsWith('WARNING (Kilotest: diagnosis disagreement received)')
+    && line.includes('Disagreements now awaiting review: 1 of 20')
+  ));
+});
+
+test('reportDisagreement records a valid disagreement to the disagreements file', async () => {
+  await response(['linkNoText', '260101T0000-mix', '0', 'alfa', 'r11', 'The element does have an accessible name']);
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  assert.equal(disagreements.length, 1);
+  assert.equal(disagreements[0].engineID, 'alfa');
+  assert.equal(disagreements[0].ruleID, 'r11');
+  assert.equal(disagreements[0].issueID, 'linkNoText');
+  assert.ok(disagreements[0].timeStamp);
+});
+
+test('reportDisagreement rejects a disagreement identical to one already on file', async () => {
+  const args = ['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name'];
+  const firstBody = await response(args);
+  const firstDetails = firstBody['response content']['details about your request'] as any;
+  assert.equal(firstDetails.error, undefined);
+  const secondBody = await response(args);
+  const secondDetails = secondBody['response content']['details about your request'] as any;
+  assert.ok(secondDetails.error.includes('identical to one already submitted'));
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  // The duplicate was not recorded a second time.
+  assert.equal(disagreements.length, 1);
+});
+
+test('reportDisagreement accepts the same reason about a different diagnosis', async () => {
+  await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'alfa', 'r11', 'The element does have an accessible name']);
+  const details = body['response content']['details about your request'] as any;
+  assert.equal(details.error, undefined);
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  assert.equal(disagreements.length, 2);
+});
+
+test('reportDisagreement creates the disagreements file when it does not exist', async () => {
+  await fs.unlink(process.env.DISAGREEMENTS_PATH!);
+  await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  assert.equal(disagreements.length, 1);
+});
+
+test('reportDisagreement rejects a disagreement with a fallback channel when the inbox is full', async () => {
+  // Fill disagreements.json with 20 distinct disagreements, reaching the cap.
+  const filler = Array.from({length: 20}, (_, i) => ({
+    timeStamp: '260101T0000',
+    issueID: 'linkNoText',
+    reportID: '260101T0000-mix',
+    catalogIndex: '0',
+    engineID: 'axe',
+    ruleID: `r${i}`,
+    reason: `Filler disagreement number ${i}, long enough to pass the length check.`
+  }));
+  await fs.writeFile(process.env.DISAGREEMENTS_PATH!, JSON.stringify(filler));
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+  const details = body['response content']['details about your request'] as any;
+  assert.ok(details.error.includes('too many disagreements are awaiting review right now.'));
+  assert.ok(details.error.includes('https://github.com/jrpool/kilotest/issues'));
+  assert.ok(details.error.includes('info@kilotest.com'));
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  // The rejected disagreement was not recorded.
+  assert.equal(disagreements.length, 20);
+});
+
+test('reportDisagreement never rejects for a full inbox when DISAGREEMENTS_MAX is 0 (no limit)', async () => {
+  process.env.DISAGREEMENTS_MAX = '0';
+  try {
+    // Fill disagreements.json past the default cap.
+    const filler = Array.from({length: 25}, (_, i) => ({
+      timeStamp: '260101T0000',
+      issueID: 'linkNoText',
+      reportID: '260101T0000-mix',
+      catalogIndex: '0',
+      engineID: 'axe',
+      ruleID: `r${i}`,
+      reason: `Filler disagreement number ${i}, long enough to pass the length check.`
+    }));
+    await fs.writeFile(process.env.DISAGREEMENTS_PATH!, JSON.stringify(filler));
+    const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+    const details = body['response content']['details about your request'] as any;
+    assert.equal(details.error, undefined);
+  }
+  finally {
+    delete process.env.DISAGREEMENTS_MAX;
+  }
+});
+
+test('reportDisagreement does not record an invalid disagreement', async () => {
+  await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'short']);
+  const disagreements = JSON.parse(await fs.readFile(process.env.DISAGREEMENTS_PATH!, 'utf8'));
+  assert.equal(disagreements.length, 0);
+});
+
+test('reportDisagreement includes tool name and metadata', async () => {
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'The element does have an accessible name']);
+  assert.equal(body['tool name'], 'reportDisagreement');
+  assert.ok(body['response metadata'].identifier);
+});
+
+test('reportDisagreement response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['linkNoText', '260101T0000-mix', '0', 'axe', 'r11', 'A reason submitted only for schema conformance.']);
+  assert.deepEqual(reportDisagreementResponseSchema.parse(body), body);
+});