@jrpool/kilotest 61.0.0 → 63.0.0

package/api/reportDisagreement.ts

@@ -0,0 +1,208 @@
+/*
+  reportDisagreement.ts
+  Processes a report that the requester disagrees with a diagnosis and returns an acknowledgement.
+*/
+
+// IMPORTS
+
+import {z} from 'zod';
+import {getResponseMetadata, getThisHost, getToolsFacts} from './util.ts';
+import {sendAlert} from '../alerts.ts';
+import {
+  checkLength,
+  describeMax,
+  getEnvMax,
+  getJSON,
+  getNowStamp,
+  getReport,
+  getTestActInstances,
+  isReportError
+} from '../util.ts';
+import {reportDisagreementResponseSchema} from './schemas.ts';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+// TYPES
+
+// The response content defined by the response schema.
+type ResponseContent = z.infer<typeof reportDisagreementResponseSchema>['response content'];
+
+// A stored disagreement with a diagnosis.
+type Disagreement = {
+  timeStamp: string;
+  issueID: string;
+  reportID: string;
+  catalogIndex: string;
+  engineID: string;
+  ruleID: string;
+  reason: string;
+};
+
+// FUNCTIONS (helpers)
+
+const getDisagreementsPath = () => process.env.DISAGREEMENTS_PATH || path.join(import.meta.dirname, '../db/disagreements.json');
+// Maximum number of diagnosis disagreements held at once. Once reached, a new, distinct
+// disagreement is rejected rather than stored, so the disagreements file and the manager
+// alert emails it generates cannot grow without bound (modeled on the feature-request
+// cap in requestFeature.ts). Configurable via DISAGREEMENTS_MAX; a value of 0 means no
+// limit.
+const getDisagreementsMax = () => getEnvMax('DISAGREEMENTS_MAX', 20);
+
+// FUNCTIONS
+
+// Returns the response body.
+export const response = async (args: string[]) => {
+  const [issueID = '', reportID = '', catalogIndex = '', engineID = '', ruleID = '', reason = ''] = args;
+  const thisHost = getThisHost();
+  const reasonCheck = checkLength(reason, 20, 1000, 'reason');
+  // Initialize the response content.
+  let responseContent: ResponseContent;
+  // If the reason is too short or too long:
+  if (reasonCheck.status === 'error') {
+    responseContent = {
+      'details about your request': {
+        error: `request invalid: your ${reasonCheck.message}`
+      }
+    };
+  }
+  // Otherwise, i.e. if it is valid:
+  else {
+    // Get the report.
+    const report = await getReport(reportID);
+    // If the report was not retrieved successfully:
+    if (isReportError(report)) {
+      responseContent = {
+        'details about your request': {
+          error: 'request invalid: the specified report does not exist or is not usable'
+        }
+      };
+    }
+    // Otherwise, if the element is not in the report's catalog:
+    else if (!report.catalog[catalogIndex]) {
+      responseContent = {
+        'details about your request': {
+          error: 'request invalid: the specified element is not in the report'
+        }
+      };
+    }
+    // Otherwise, i.e. if the report and element exist:
+    else {
+      // Get the reported diagnoses of the element for the issue matching the engine and
+      // rule (a ruleID is only unique within a rule engine: axe and alfa, for example,
+      // both report rules numbered like r11 for the same element and issue).
+      const matches = getTestActInstances(report, {violationsOnly: true, issueID, catalogIndex})
+      .filter(({act, instance}) => act.which === engineID && instance.ruleID === ruleID);
+      // If no reported diagnosis matches:
+      if (!matches.length) {
+        responseContent = {
+          'details about your request': {
+            error: 'request invalid: no such diagnosis was reported for the specified element and issue'
+          }
+        };
+      }
+      // Otherwise, i.e. if the diagnosis exists:
+      else {
+        // Get the disagreements already on file.
+        const disagreementsPath = getDisagreementsPath();
+        let disagreements: Disagreement[] = [];
+        try {
+          const existing = await fs.readFile(disagreementsPath, 'utf8');
+          disagreements = JSON.parse(existing);
+        }
+        catch {
+          // Initialize an empty disagreements array.
+        }
+        // If the disagreement is identical to one already on file:
+        if (disagreements.some(d =>
+          d.issueID === issueID
+          && d.reportID === reportID
+          && d.catalogIndex === catalogIndex
+          && d.engineID === engineID
+          && d.ruleID === ruleID
+          && d.reason === reason
+        )) {
+          responseContent = {
+            'details about your request': {
+              error: 'request invalid: your disagreement is identical to one already submitted, ' +
+                'but you are welcome to submit a different one'
+            }
+          };
+        }
+        // Otherwise, if the disagreements already on file have reached the cap (a cap of 0
+        // means no limit, so the queue is never full):
+        else if (getDisagreementsMax() > 0 && disagreements.length >= getDisagreementsMax()) {
+          responseContent = {
+            'details about your request': {
+              error: 'request invalid: too many disagreements are awaiting review right now. ' +
+                'Please try again later, or post your report at https://github.com/jrpool/kilotest/issues ' +
+                'or email info@kilotest.com.'
+            }
+          };
+        }
+        // Otherwise, i.e. if it is not a duplicate and the queue is not full:
+        else {
+          // Record the disagreement alongside the existing ones.
+          disagreements.push({
+            timeStamp: getNowStamp(),
+            issueID,
+            reportID,
+            catalogIndex,
+            engineID,
+            ruleID,
+            reason
+          });
+          await fs.mkdir(path.dirname(disagreementsPath), {recursive: true});
+          await fs.writeFile(disagreementsPath, getJSON(disagreements));
+          // Notify the manager, including the resulting count, so a maintainer who has been
+          // away sees at a glance how urgently the disagreements need review. The ruleID is
+          // truncated because some rule engines (nuVal, nuVnu) use entire validator messages
+          // as rule identifiers, which can be arbitrarily long.
+          await sendAlert(
+            'Kilotest: diagnosis disagreement received',
+            `Disputed diagnosis: ${engineID} rule ${ruleID.slice(0, 500)} classified as issue ${issueID} ` +
+            `on element ${catalogIndex} of report ${reportID}\nReason: ${reason}\n` +
+            `Disagreements now awaiting review: ${disagreements.length} of ${describeMax(getDisagreementsMax())}`
+          );
+          // Add the disposition to the response content.
+          responseContent = {
+            'details about your request': {
+              'date and time received': new Date().toISOString(),
+              'diagnosis you disagree with': {
+                'issue identifier': issueID,
+                'identifier of the rule engine': engineID,
+                'identifier of the violated rule': ruleID
+              },
+              disposition: 'received and logged; manager notified'
+            }
+          };
+        }
+      }
+    }
+  }
+  // Create a response body.
+  const body = {
+    'tool collection': getToolsFacts(),
+    'tool name': 'reportDisagreement',
+    'this request': {
+      description: 'Process my report that I disagree with a diagnosis. The issueID, reportID, and catalogIndex parameters identify the issue, report, and element of the diagnosis. They were in the response to my earlier listDiagnoses request. The engineID and ruleID properties of the request body identify which diagnosis I disagree with, and the reason property explains why I disagree with it.',
+      method: 'POST',
+      URL: `${thisHost}/api/reportDisagreement/${issueID}/${reportID}/${catalogIndex}`,
+      body: {
+        engineID,
+        ruleID,
+        reason
+      },
+      'closest ancestor request': {
+        'tool name': 'listDiagnoses',
+        description: 'Provide details about one element reported as exhibiting one issue in one report, including the diagnoses provided by rule engines about how the element exhibited the issue.',
+        method: 'GET',
+        URL: `${thisHost}/api/listDiagnoses/${catalogIndex}/${issueID}/${reportID}`
+      }
+    },
+    'URLs of similar requests for web users': null,
+    'response metadata': getResponseMetadata(),
+    'response content': responseContent
+  };
+  // Return it.
+  return body;
+};

package/api/requestFeature.test.ts

@@ -37,6 +37,7 @@ beforeEach(async () => {
 });
 
 import {response} from './requestFeature.ts';
+import {requestFeatureResponseSchema} from './schemas.ts';
 
 after(async () => {
   console.log = originalLog;
@@ -186,3 +187,11 @@ test('requestFeature accepts a feature request that differs from an existing one
   const featureRequests = JSON.parse(await fs.readFile(process.env.FEATURE_REQUESTS_PATH!, 'utf8'));
   assert.equal(featureRequests.length, 2);
 });
+
+test('requestFeature response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['A feature request submitted only for schema conformance.']);
+  assert.deepEqual(requestFeatureResponseSchema.parse(body), body);
+});

package/api/requestNewTest.test.ts

@@ -42,6 +42,7 @@ beforeEach(async () => {
 });
 
 import {response} from './requestNewTest.ts';
+import {requestNewTestResponseSchema} from './schemas.ts';
 
 after(() => {
   console.log = originalLog;
@@ -208,3 +209,11 @@ test('requestNewTest rejects a request with a fallback channel when the queue is
   assert.ok(disposition['what happens next'].includes('https://github.com/jrpool/kilotest/issues'));
   assert.ok(disposition['what happens next'].includes('info@kilotest.com'));
 });
+
+test('requestNewTest response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['Schema Conformance Page', 'https://example.com/schema-conformance', 'A reason that is long enough.']);
+  assert.deepEqual(requestNewTestResponseSchema.parse(body), body);
+});

package/api/requestRetest.test.ts

@@ -56,6 +56,7 @@ beforeEach(async () => {
 });
 
 import {response} from './requestRetest.ts';
+import {requestRetestResponseSchema} from './schemas.ts';
 
 after(async () => {
   console.log = originalLog;
@@ -193,3 +194,11 @@ test('requestRetest rejects a request with a fallback channel when the queue is
   assert.ok(disposition['what happens next'].includes('https://github.com/jrpool/kilotest/issues'));
   assert.ok(disposition['what happens next'].includes('info@kilotest.com'));
 });
+
+test('requestRetest response conforms to its response schema', async () => {
+  // parse() catches missing or mistyped keys; comparing the parsed result to the
+  // original catches extra keys that zod strips but that the strict JSON schema
+  // advertised to MCP clients (additionalProperties: false) rejects.
+  const body = await response(['260101T0003-ret', 'A reason that is long enough.']);
+  assert.deepEqual(requestRetestResponseSchema.parse(body), body);
+});

package/api/routes.test.ts

@@ -11,9 +11,9 @@ import {routes} from './routes.ts';
 
 // TESTS
 
-test('routes exports an array with 11 operations', () => {
+test('routes exports an array with 12 operations', () => {
   assert.ok(Array.isArray(routes));
-  assert.equal(routes.length, 11);
+  assert.equal(routes.length, 12);
 });
 
 test('routes includes all expected operation IDs', () => {
@@ -27,6 +27,7 @@ test('routes includes all expected operation IDs', () => {
     'listViolators',
     'orderNewTest',
     'orderRetest',
+    'reportDisagreement',
     'requestFeature',
     'requestNewTest',
     'requestRetest'

package/api/routes.ts

@@ -17,6 +17,7 @@ import {
   orderNewTestSchema,
   orderRetestSchema,
   awaitTestSchema,
+  reportDisagreementSchema,
   listPagesResponseSchema,
   listIssuesResponseSchema,
   listViolatorsResponseSchema,
@@ -27,7 +28,8 @@ import {
   requestFeatureResponseSchema,
   orderNewTestResponseSchema,
   orderRetestResponseSchema,
-  awaitTestResponseSchema
+  awaitTestResponseSchema,
+  reportDisagreementResponseSchema
 } from './schemas.ts';
 
 // ROUTES
@@ -99,6 +101,23 @@ export const routes = [
     bodySchema: z.object(requestFeatureSchema),
     responseSchema: requestFeatureResponseSchema
   },
+  {
+    operationId: 'reportDisagreement',
+    method: 'post',
+    path: '/api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}',
+    summary: 'Report disagreement with a diagnosis',
+    pathParamsSchema: z.object({
+      issueID: reportDisagreementSchema.issueID,
+      reportID: reportDisagreementSchema.reportID,
+      catalogIndex: reportDisagreementSchema.catalogIndex
+    }),
+    bodySchema: z.object({
+      engineID: reportDisagreementSchema.engineID,
+      ruleID: reportDisagreementSchema.ruleID,
+      reason: reportDisagreementSchema.reason
+    }),
+    responseSchema: reportDisagreementResponseSchema
+  },
   {
     operationId: 'orderNewTest',
     method: 'post',

package/api/schemas.ts

@@ -73,6 +73,16 @@ export const awaitTestSchema = {
   reportID: z.string().describe('Report identifier returned by orderNewTest or orderRetest (example: 260503T0432-x9z)')
 };
 
+// reportDisagreement: POST /api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}
+export const reportDisagreementSchema = {
+  issueID: z.string().describe('Issue identifier (example: contrastPoor)'),
+  reportID: z.string().describe('Identifier of the report (example: 260503T0432-x9z)'),
+  catalogIndex: z.string().describe('Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'),
+  engineID: z.string().describe('Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output'),
+  ruleID: z.string().describe('Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead'),
+  reason: z.string().describe('20- to 1000-character explanation of why the diagnosis is disputed')
+};
+
 // listPages takes no input; omitted (mcp.ts already uses inputSchema: {}).
 
 // RESPONSE SCHEMAS
@@ -85,7 +95,7 @@ const ruleEngineFactsSchema = z.object({
 }).meta({id: 'RuleEngineFacts'});
 
 // Facts about the Kilotest tool collection, included in every response.
-const toolsFactsSchema = z.object({
+export const toolsFactsSchema = z.object({
   name: z.string(),
   description: z.object({
     'what Kilotest does': z.string(),
@@ -98,8 +108,10 @@ const toolsFactsSchema = z.object({
     'how to generate more findings': z.object({
       'new testing': z.string(),
       'retesting': z.string(),
+      'approval': z.string(),
       'latency': z.string(),
-      'confirmation': z.string()
+      'request confirmation': z.string(),
+      'order confirmation': z.string()
     })
   }).describe('Explanation of what Kilotest does and how to use its tools.'),
   URL: z.string().describe('URL of the Kilotest MCP server.'),
@@ -278,7 +290,7 @@ export const listViolatorsResponseSchema = envelope('GET', z.object({
     'tag name': z.unknown().describe('Copied from the report catalog entry; type not guaranteed.'),
     'inner text': z.unknown().describe('Copied from the report catalog entry; type not guaranteed.'),
     'count of rule engines reporting that the element exhibited the issue': z.number(),
-    'how to get details about the element': z.object({URL: z.string(), 'request method': z.literal('GET')}),
+    'how to get details about the element': z.object({method: z.literal('GET'), URL: z.string()}),
     'web users can get details about the element at': z.string()
   })).nullable()
 }));
@@ -297,10 +309,20 @@ export const listDiagnosesResponseSchema = envelope('GET', z.object({
     'x, y, width, and height of bounding box': z.unknown().describe('Copied from the report catalog entry; type not guaranteed.')
   }).nullable(),
   'diagnoses of how the element exhibited the issue': z.array(z.object({
+    'identifier of the rule engine': z.unknown().describe('Copied from the report act (which); type not guaranteed.'),
     'identifier of the violated rule': z.unknown().describe('Copied from the report instance (ruleID); type not guaranteed.'),
     'description of the violation': z.unknown().describe('Copied from the report instance (what); type not guaranteed.'),
     'severity of the violation on a 0-to-3 scale': z.unknown().describe('Copied from the report instance (ordinalSeverity); type not guaranteed.'),
-    'count of violations of the rule by the element': z.number()
+    'count of violations of the rule by the element': z.number(),
+    'how to report that you disagree with this diagnosis': z.object({
+      method: z.literal('POST'),
+      URL: z.string(),
+      'request body': z.object({
+        engineID: z.string(),
+        ruleID: z.string(),
+        reason: z.string()
+      })
+    })
   })).or(z.object({error: z.string()})).nullable()
 }));
 
@@ -362,6 +384,28 @@ export const requestFeatureResponseSchema = envelope(
   z.null()
 );
 
+export const reportDisagreementResponseSchema = envelope(
+  'POST',
+  z.object({'details about your request': z.union([
+    z.object({error: z.string()}),
+    z.object({
+      'date and time received': z.string(),
+      'diagnosis you disagree with': z.object({
+        'issue identifier': z.string(),
+        'identifier of the rule engine': z.string(),
+        'identifier of the violated rule': z.string()
+      }),
+      disposition: z.string()
+    })
+  ])}),
+  z.object({
+    engineID: reportDisagreementSchema.engineID,
+    ruleID: reportDisagreementSchema.ruleID,
+    reason: reportDisagreementSchema.reason
+  }),
+  z.null()
+);
+
 // Identifies the report a successfully ordered or awaited job will produce, once complete.
 const reportIdentifierSchema = z.string().describe('Identifier of the report, to pass to listIssues once the report is ready (example: 260503T0432-x9z).');
 

package/api/util.test.ts

@@ -17,6 +17,7 @@ for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_A
 }
 
 import {getReportBasics, getResponseMetadata, getRuleEngineFacts, getRuleEnginesFacts, getIssueSpec, getToolsFacts, getRequestFailureReason} from './util.ts';
+import {toolsFactsSchema} from './schemas.ts';
 import {processTestRequest} from '../util.ts';
 
 // SETUP
@@ -137,6 +138,14 @@ test('getToolsFacts returns facts about the Kilotest tool collection', () => {
   assert.ok(facts.description['how to generate more findings']);
 });
 
+test('getToolsFacts output conforms to toolsFactsSchema', () => {
+  // parse() catches missing or mistyped keys, and comparing the parsed result to
+  // the original catches extra keys that zod strips but that the strict JSON
+  // schema advertised to MCP clients (additionalProperties: false) rejects.
+  const facts = getToolsFacts();
+  assert.deepEqual(toolsFactsSchema.parse(facts), facts);
+});
+
 test('getRequestFailureReason returns the URL-length reason for a urlLength result', () => {
   // processTestRequest can return urlLength for a new-test request whose URL is
   // outside the 12-to-300-character bound. No current caller reaches this

package/apiIntegration.test.ts

@@ -16,11 +16,13 @@ import {requestHandler} from './index.ts';
 
 const port = 3998;
 const featureRequestsPath = path.join(import.meta.dirname, 'test/fixtures/apiIntegration/featureRequests.json');
+const disagreementsPath = path.join(import.meta.dirname, 'test/fixtures/apiIntegration/disagreements.json');
 
 // SETUP AND TEARDOWN
 
 const savedDBDir = process.env.DB_DIR;
 const savedFeatureRequestsPath = process.env.FEATURE_REQUESTS_PATH;
+const savedDisagreementsPath = process.env.DISAGREEMENTS_PATH;
 let server: http.Server;
 
 before(async () => {
@@ -31,6 +33,9 @@ before(async () => {
   process.env.FEATURE_REQUESTS_PATH = featureRequestsPath;
   await fs.mkdir(path.dirname(featureRequestsPath), {recursive: true});
   await fs.writeFile(featureRequestsPath, '[]\n');
+  // The same for diagnosis disagreements.
+  process.env.DISAGREEMENTS_PATH = disagreementsPath;
+  await fs.writeFile(disagreementsPath, '[]\n');
   server = http.createServer(requestHandler);
   await new Promise<void>(resolve => server.listen(port, () => resolve()));
 });
@@ -48,6 +53,12 @@ after(async () => {
   else {
     delete process.env.FEATURE_REQUESTS_PATH;
   }
+  if (savedDisagreementsPath !== undefined) {
+    process.env.DISAGREEMENTS_PATH = savedDisagreementsPath;
+  }
+  else {
+    delete process.env.DISAGREEMENTS_PATH;
+  }
   await fs.rm(path.dirname(featureRequestsPath), {recursive: true, force: true});
   server.closeAllConnections?.();
   await new Promise<void>(resolve => {
@@ -106,3 +117,14 @@ test('POST /api/requestFeature parses a JSON body and returns a JSON response th
   assert.equal(details.error, undefined);
   assert.equal(details.disposition, 'received and logged; manager notified');
 });
+
+test('POST /api/reportDisagreement parses path parameters and a JSON body and returns a JSON response through the HTTP routing layer', async () => {
+  const {body} = await request('POST', '/api/reportDisagreement/linkNoText/260101T0000-mix/0', {
+    engineID: 'axe',
+    ruleID: 'r11',
+    reason: 'The element does have an accessible name'
+  });
+  const details = body['response content']['details about your request'];
+  assert.equal(details.error, undefined);
+  assert.equal(details.disposition, 'received and logged; manager notified');
+});

package/docs/SERVICE.md

@@ -409,9 +409,11 @@ Alerts are sent to the maintainer in the following conditions:
 
 7. **MCP feature request received**: When a user submits a feature request via the MCP server's `requestFeature` tool, an alert is sent with subject `"MCP feature request received"`, including the text of the feature request, and the request is also appended to `db/featureRequests.json`, the same way tutorial comments are appended to their own files (see item 1), so that a manager can review or delete it later via `/deleteNotesForm.html`.
 
-8. **WAVE balance low**: When Kilotest detects that the account balance for the WAVE API service (used for accessibility testing) has fallen below a threshold, an alert is sent with subject `"Kilotest: WAVE balance low"`, indicating the number of credits remaining.
+8. **Diagnosis disagreement received**: When a user reports disagreement with a diagnosis via the API’s or MCP server’s `reportDisagreement` operation, an alert is sent with subject `"Kilotest: diagnosis disagreement received"`, including the reason, the disputed diagnosis (rule engine, rule, issue, element, and report), and the disagreement is also appended to `db/disagreements.json`, so that a manager can review it and, when warranted, propose a correction to the `testaro-issues` classification (see item 3 for how such corrections take effect).
 
-9. **AI service balance low**: When Kilotest detects that the account balance for an AI service account (such as Anthropic) has fallen below a threshold, an alert is sent with subject `"Kilotest: AI service [N] balance low"`, indicating the approximate remaining balance in dollars and the estimated cost per job.
+9. **WAVE balance low**: When Kilotest detects that the account balance for the WAVE API service (used for accessibility testing) has fallen below a threshold, an alert is sent with subject `"Kilotest: WAVE balance low"`, indicating the number of credits remaining.
+
+10. **AI service balance low**: When Kilotest detects that the account balance for an AI service account (such as Anthropic) has fallen below a threshold, an alert is sent with subject `"Kilotest: AI service [N] balance low"`, indicating the approximate remaining balance in dollars and the estimated cost per job.
 
 ### Alert configuration
 
@@ -427,7 +429,7 @@ If any of these five variables is missing or empty at alert time, the alert is n
 
 ### Inbox size limits
 
-Four submission types (test requests and retest requests together, web tutorial comments, AI tutorial comments, and MCP feature requests) are each held in a pending "inbox" (`db/jobs/testRequests.json`, `db/comments/tutorialWeb.json`, `db/comments/tutorialAI.json`, and `db/featureRequests.json` respectively) awaiting manual review by the maintainer. Each inbox has a maximum size; once reached, a new, distinct submission is rejected (with a message suggesting the requester instead post to the [GitHub issues page](https://github.com/jrpool/kilotest/issues) or email `info@kilotest.com`) rather than added, so neither the inbox file nor the manager alert emails it generates can grow without bound (see GitHub issue #3, abuse type 3, repeated/automated submissions). This is modeled on a full mailbox rejecting new mail, rather than on a submission rate limit: it is the backlog size itself, not how quickly it grows, that costs the maintainer review time and risks unbounded storage. Every alert sent for an accepted submission reports the resulting inbox count against its cap, so a maintainer returning after an absence can immediately see how close an inbox is to full.
+Five submission types (test requests and retest requests together, web tutorial comments, AI tutorial comments, MCP feature requests, and diagnosis disagreements) are each held in a pending "inbox" (`db/jobs/testRequests.json`, `db/comments/tutorialWeb.json`, `db/comments/tutorialAI.json`, `db/featureRequests.json`, and `db/disagreements.json` respectively) awaiting manual review by the maintainer. Each inbox has a maximum size; once reached, a new, distinct submission is rejected (with a message suggesting the requester instead post to the [GitHub issues page](https://github.com/jrpool/kilotest/issues) or email `info@kilotest.com`) rather than added, so neither the inbox file nor the manager alert emails it generates can grow without bound (see GitHub issue #3, abuse type 3, repeated/automated submissions). This is modeled on a full mailbox rejecting new mail, rather than on a submission rate limit: it is the backlog size itself, not how quickly it grows, that costs the maintainer review time and risks unbounded storage. Every alert sent for an accepted submission reports the resulting inbox count against its cap, so a maintainer returning after an absence can immediately see how close an inbox is to full.
 
 Each cap defaults to 20 and can be set independently via an environment variable:
 
@@ -435,6 +437,7 @@ Each cap defaults to 20 and can be set independently via an environment variable
 2. **`TUTORIAL_WEB_COMMENTS_MAX`**: The maximum number of comments held on the web tutorial page (`/tutorialWeb.html`).
 3. **`TUTORIAL_AI_COMMENTS_MAX`**: The maximum number of comments held on the AI agent tutorial page (`/tutorialAI.html`).
 4. **`FEATURE_REQUESTS_MAX`**: The maximum number of feature requests held via the MCP server's `requestFeature` tool.
+5. **`DISAGREEMENTS_MAX`**: The maximum number of disagreements with diagnoses held via the API’s or MCP server’s `reportDisagreement` operation.
 
 Each variable, if set, must be a non-negative integer; if unset, empty, or not a non-negative integer, the default of 20 is used instead. A value of `0` means no limit: that inbox is never treated as full, regardless of size. Unlike the alert-configuration variables above, these are not required: an unset value never disables the limit, it only selects the default size.
 

package/env.example

@@ -35,6 +35,8 @@ TUTORIAL_WEB_COMMENTS_PATH=__placeholder__
 TUTORIAL_AI_COMMENTS_PATH=__placeholder__
 # Path of the feature requests file (defaults to db/featureRequests.json).
 FEATURE_REQUESTS_PATH=__placeholder__
+# Path of the diagnosis disagreements file (defaults to db/disagreements.json).
+DISAGREEMENTS_PATH=__placeholder__
 
 # Inbox size limits: each defaults to 20 if unset, empty, or not a non-negative integer.
 # A value of 0 means no limit.
@@ -47,6 +49,9 @@ TUTORIAL_WEB_COMMENTS_MAX=__placeholder__
 TUTORIAL_AI_COMMENTS_MAX=__placeholder__
 # Maximum number of feature requests held via the MCP server's requestFeature tool.
 FEATURE_REQUESTS_MAX=__placeholder__
+# Maximum number of disagreements with diagnoses held via the MCP server's
+# reportDisagreement tool.
+DISAGREEMENTS_MAX=__placeholder__
 # Maximum number of jobs allowed in the job queue (db/jobs/queue) at once. Shared by every
 # path that enqueues a job: manager approval of a pending test/retest request, and the
 # orderNewTest/orderRetest MCP tools, which enqueue directly.

package/index.test.ts

@@ -2334,6 +2334,50 @@ test('startServer starts an HTTPS server when protocol is https', {timeout: 2000
   }
 });
 
+test('startServer throws when THIS_KILOTEST_HOST is unset', async () => {
+  const savedHost = process.env.THIS_KILOTEST_HOST;
+  const savedProtocol = process.env.PROTOCOL;
+  delete process.env.THIS_KILOTEST_HOST;
+  delete process.env.PROTOCOL;
+  try {
+    await assert.rejects(startServer(), /THIS_KILOTEST_HOST/);
+  }
+  finally {
+    if (savedHost !== undefined) {
+      process.env.THIS_KILOTEST_HOST = savedHost;
+    }
+    if (savedProtocol !== undefined) {
+      process.env.PROTOCOL = savedProtocol;
+    }
+  }
+});
+
+test('startServer throws when PROTOCOL is https and KEY and CERT are unset', async () => {
+  const savedProtocol = process.env.PROTOCOL;
+  const savedKey = process.env.KEY;
+  const savedCert = process.env.CERT;
+  process.env.PROTOCOL = 'https';
+  delete process.env.KEY;
+  delete process.env.CERT;
+  try {
+    await assert.rejects(startServer(), /KEY.*CERT|CERT.*KEY/);
+  }
+  finally {
+    if (savedProtocol !== undefined) {
+      process.env.PROTOCOL = savedProtocol;
+    }
+    else {
+      delete process.env.PROTOCOL;
+    }
+    if (savedKey !== undefined) {
+      process.env.KEY = savedKey;
+    }
+    if (savedCert !== undefined) {
+      process.env.CERT = savedCert;
+    }
+  }
+});
+
 test('runIfMain calls startServer when mainModule matches currentModule', async () => {
   let called = false;
   const starter = async () => {
@@ -2358,3 +2402,25 @@ test('runIfMain does not call startServer when mainModule differs from currentMo
   await new Promise<void>(resolve => setImmediate(resolve));
   assert.equal(called, false);
 });
+
+test('runIfMain logs the error and sets a failing exit code when the starter rejects', async () => {
+  const savedExitCode = process.exitCode;
+  const logged: string[] = [];
+  const originalLog = console.log;
+  console.log = (line: string) => logged.push(line);
+  const starter = async () => {
+    throw new Error('injected startup failure');
+  };
+  const fakeModule = {};
+  try {
+    runIfMain(fakeModule, fakeModule, starter);
+    // Wait for the rejection handler's microtasks to run.
+    await new Promise<void>(resolve => setImmediate(resolve));
+    assert.ok(logged.some(line => line.includes('injected startup failure')));
+    assert.equal(process.exitCode, 1);
+  }
+  finally {
+    console.log = originalLog;
+    process.exitCode = savedExitCode;
+  }
+});