@jrpool/kilotest 61.0.0 → 63.0.0

package/llms-full.txt

@@ -124,6 +124,19 @@ Kilotest offers these tools:
   - path: `/api/requestFeature`
   - payload properties
     - `feature`: description and motivation of the feature improvement or the new feature
+- `reportDisagreement`
+  - method: `POST`
+  - purpose: report disagreement with a diagnosis shown by `listDiagnoses` of how an element exhibited an issue
+  - path: `/api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}`
+  - parameters
+    - `issueID`: identifier of the issue
+    - `reportID`: identifier of the report, with YYMMDDTHHmm format for its initial segment (example: 260503T0432-x9z)
+    - `catalogIndex`: index of the catalog entry of the element
+  - source of parameters: response to a `listDiagnoses` request
+  - payload properties
+    - `engineID`: identifier of the rule engine that produced the disputed diagnosis, shown in the `listDiagnoses` response
+    - `ruleID`: identifier of the rule whose diagnosis is disputed, shown in the `listDiagnoses` response, or, if it is null, the description of the violation
+    - `reason`: 20- to 1000-character explanation of why the diagnosis is disputed
 - `orderNewTest`
   - method: `POST`
   - purpose: order immediate, automatically approved testing of one web page about which no report is available, instead of waiting for manual approval

package/llms.txt

@@ -12,6 +12,7 @@ Kilotest integrates an ensemble of 12 rule engines, with about 1,300 rules, to t
 - Request testing of a web page: `/api/requestNewTest` (`POST`)
 - Request retesting of a web page: `/api/requestRetest/{reportID}` (`POST`)
 - Request adding or improving a feature: `/api/requestFeature` (`POST`)
+- Report disagreement with a diagnosis: `/api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}` (`POST`)
 - Order immediate, automatically approved testing of a web page: `/api/orderNewTest` (`POST`)
 - Order immediate, automatically approved retesting of a web page: `/api/orderRetest/{reportID}` (`POST`)
 - Wait for a job ordered via orderNewTest or orderRetest to complete: `/api/awaitTest/{reportID}` (`POST`)

package/mcp.test.ts

@@ -37,10 +37,10 @@ test('mcpPath is /mcp', () => {
   assert.equal(mcpPath, '/mcp');
 });
 
-test('createMCPServer registers all 12 tools', () => {
+test('createMCPServer registers all 13 tools', () => {
   const server = createMCPServer();
   const toolNames = Object.keys((server as any)._registeredTools);
-  assert.equal(toolNames.length, 12);
+  assert.equal(toolNames.length, 13);
   assert.deepEqual(toolNames, [
     'getKilotestOverview',
     'listPages',
@@ -51,6 +51,7 @@ test('createMCPServer registers all 12 tools', () => {
     'requestNewTest',
     'requestRetest',
     'requestFeature',
+    'reportDisagreement',
     'orderNewTest',
     'orderRetest',
     'awaitTest'
@@ -197,6 +198,23 @@ test('requestFeature handler returns content and structuredContent', async () =>
   assert.equal(getToolCallCount('requestFeature'), countBefore + 1);
 });
 
+test('reportDisagreement handler returns content and structuredContent', async () => {
+  const server = createMCPServer();
+  const countBefore = getToolCallCount('reportDisagreement');
+  const result = await (server as any)._registeredTools.reportDisagreement.handler({
+    issueID: 'linkNoText',
+    reportID: '260101T0000-mix',
+    catalogIndex: '0',
+    engineID: 'axe',
+    ruleID: 'r11',
+    reason: 'The element does have an accessible name'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+  assert.equal(getToolCallCount('reportDisagreement'), countBefore + 1);
+});
+
 test('orderNewTest handler returns content and structuredContent, and enqueues a job', async () => {
   const {jobsPath} = await import('./util.ts');
   const server = createMCPServer();
@@ -349,7 +367,7 @@ test('handleMCP responds to initialize with server info', async () => {
   }
 });
 
-test('handleMCP lists all 12 tools via tools/list', async () => {
+test('handleMCP lists all 13 tools via tools/list', async () => {
   const server = await startMCPServer();
   try {
     const port = server.address().port;
@@ -357,7 +375,7 @@ test('handleMCP lists all 12 tools via tools/list', async () => {
     assert.equal(res.statusCode, 200);
     const result = parseSSEResult(res.body);
     const toolNames = result.result.tools.map((t: any) => t.name);
-    assert.equal(toolNames.length, 12);
+    assert.equal(toolNames.length, 13);
     assert.deepEqual(toolNames, [
       'getKilotestOverview',
       'listPages',
@@ -368,6 +386,7 @@ test('handleMCP lists all 12 tools via tools/list', async () => {
       'requestNewTest',
       'requestRetest',
       'requestFeature',
+      'reportDisagreement',
       'orderNewTest',
       'orderRetest',
       'awaitTest'

package/mcp.ts

@@ -19,6 +19,7 @@ import * as requestFeatureAPI from './api/requestFeature.ts';
 import * as orderNewTestAPI from './api/orderNewTest.ts';
 import * as orderRetestAPI from './api/orderRetest.ts';
 import * as awaitTestAPI from './api/awaitTest.ts';
+import * as reportDisagreementAPI from './api/reportDisagreement.ts';
 import {version} from './api/version.ts';
 import {recordMetric} from './util.ts';
 
@@ -33,6 +34,7 @@ import {
   orderNewTestSchema,
   orderRetestSchema,
   awaitTestSchema,
+  reportDisagreementSchema,
   listPagesResponseSchema,
   listIssuesResponseSchema,
   listViolatorsResponseSchema,
@@ -43,7 +45,8 @@ import {
   requestFeatureResponseSchema,
   orderNewTestResponseSchema,
   orderRetestResponseSchema,
-  awaitTestResponseSchema
+  awaitTestResponseSchema,
+  reportDisagreementResponseSchema
 } from './api/schemas.ts';
 
 // CONSTANTS
@@ -257,6 +260,26 @@ export const createMCPServer = (): McpServer => {
       return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result};
     }
   );
+  server.registerTool(
+    'reportDisagreement',
+    {
+      description: 'Reports that you disagree with a diagnosis shown by the listDiagnoses tool; queued for manager review. Use this after listDiagnoses when you believe a rule violation was incorrectly diagnosed or classified into an issue, providing the engineID and ruleID shown in the diagnosis and your reason. Returns a confirmation with disposition details.',
+      inputSchema: reportDisagreementSchema,
+      outputSchema: reportDisagreementResponseSchema,
+      annotations: {
+        title: 'Report disagreement with a diagnosis',
+        readOnlyHint: false,
+        idempotentHint: false,
+        destructiveHint: false,
+        openWorldHint: false
+      }
+    },
+    async ({issueID, reportID, catalogIndex, engineID, ruleID, reason}) => {
+      const result = await reportDisagreementAPI.response([issueID, reportID, catalogIndex, engineID, ruleID, reason]);
+      await recordMetric('mcpToolCalls', 'reportDisagreement');
+      return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result};
+    }
+  );
   server.registerTool(
     'orderNewTest',
     {

package/openapi-3.0.yaml

@@ -2,7 +2,7 @@ openapi: 3.0.3
 info:
   title: Kilotest API
   description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and web standards, including HTML, CSS, ARIA, and WCAG). Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access. With this API you can request that a page be tested or retested, discover available reports, incrementally retrieve facts from a report: from what reports are available, to what issues were reported in one report, to what elements violated one issue, to what diagnoses rule engines gave for one violation.'
-  version: 61.0.0
+  version: 63.0.0
   license:
     name: MIT
     url: https://github.com/jrpool/kilotest/blob/main/LICENSE
@@ -560,15 +560,15 @@ paths:
                             how to get details about the element:
                               type: object
                               properties:
-                                URL:
-                                  type: string
-                                request method:
+                                method:
                                   type: string
                                   enum:
                                     - GET
+                                URL:
+                                  type: string
                               required:
+                                - method
                                 - URL
-                                - request method
                               additionalProperties: false
                             web users can get details about the element at:
                               type: string
@@ -713,6 +713,8 @@ paths:
                                 items:
                                   type: object
                                   properties:
+                                    identifier of the rule engine:
+                                      description: Copied from the report act (which); type not guaranteed.
                                     identifier of the violated rule:
                                       description: Copied from the report instance (ruleID); type not guaranteed.
                                     description of the violation:
@@ -721,11 +723,41 @@ paths:
                                       description: Copied from the report instance (ordinalSeverity); type not guaranteed.
                                     count of violations of the rule by the element:
                                       type: number
+                                    how to report that you disagree with this diagnosis:
+                                      type: object
+                                      properties:
+                                        method:
+                                          type: string
+                                          enum:
+                                            - POST
+                                        URL:
+                                          type: string
+                                        request body:
+                                          type: object
+                                          properties:
+                                            engineID:
+                                              type: string
+                                            ruleID:
+                                              type: string
+                                            reason:
+                                              type: string
+                                          required:
+                                            - engineID
+                                            - ruleID
+                                            - reason
+                                          additionalProperties: false
+                                      required:
+                                        - method
+                                        - URL
+                                        - request body
+                                      additionalProperties: false
                                   required:
+                                    - identifier of the rule engine
                                     - identifier of the violated rule
                                     - description of the violation
                                     - severity of the violation on a 0-to-3 scale
                                     - count of violations of the rule by the element
+                                    - how to report that you disagree with this diagnosis
                                   additionalProperties: false
                               - type: object
                                 properties:
@@ -1191,6 +1223,156 @@ paths:
                   - response metadata
                   - response content
                 additionalProperties: false
+  /api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}:
+    post:
+      operationId: reportDisagreement
+      summary: Report disagreement with a diagnosis
+      parameters:
+        - in: path
+          name: issueID
+          schema:
+            type: string
+            description: 'Issue identifier (example: contrastPoor)'
+          required: true
+          description: 'Issue identifier (example: contrastPoor)'
+        - in: path
+          name: reportID
+          schema:
+            type: string
+            description: 'Identifier of the report (example: 260503T0432-x9z)'
+          required: true
+          description: 'Identifier of the report (example: 260503T0432-x9z)'
+        - in: path
+          name: catalogIndex
+          schema:
+            type: string
+            description: 'Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'
+          required: true
+          description: 'Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                engineID:
+                  type: string
+                  description: 'Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output'
+                ruleID:
+                  type: string
+                  description: 'Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead'
+                reason:
+                  type: string
+                  description: 20- to 1000-character explanation of why the diagnosis is disputed
+              required:
+                - engineID
+                - ruleID
+                - reason
+      responses:
+        '200':
+          description: Report disagreement with a diagnosis
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tool collection:
+                    $ref: '#/components/schemas/ToolsFacts'
+                  tool name:
+                    type: string
+                  this request:
+                    type: object
+                    properties:
+                      description:
+                        type: string
+                      method:
+                        type: string
+                        enum:
+                          - POST
+                      URL:
+                        type: string
+                      body:
+                        type: object
+                        properties:
+                          engineID:
+                            type: string
+                            description: 'Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output'
+                          ruleID:
+                            type: string
+                            description: 'Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead'
+                          reason:
+                            type: string
+                            description: 20- to 1000-character explanation of why the diagnosis is disputed
+                        required:
+                          - engineID
+                          - ruleID
+                          - reason
+                        additionalProperties: false
+                      closest ancestor request:
+                        allOf:
+                          - $ref: '#/components/schemas/RequestReference'
+                        type: object
+                        nullable: true
+                    required:
+                      - description
+                      - method
+                      - URL
+                      - body
+                      - closest ancestor request
+                    additionalProperties: false
+                  URLs of similar requests for web users:
+                    type: object
+                    nullable: true
+                  response metadata:
+                    $ref: '#/components/schemas/ResponseMetadata'
+                  response content:
+                    type: object
+                    properties:
+                      details about your request:
+                        anyOf:
+                          - type: object
+                            properties:
+                              error:
+                                type: string
+                            required:
+                              - error
+                            additionalProperties: false
+                          - type: object
+                            properties:
+                              date and time received:
+                                type: string
+                              diagnosis you disagree with:
+                                type: object
+                                properties:
+                                  issue identifier:
+                                    type: string
+                                  identifier of the rule engine:
+                                    type: string
+                                  identifier of the violated rule:
+                                    type: string
+                                required:
+                                  - issue identifier
+                                  - identifier of the rule engine
+                                  - identifier of the violated rule
+                                additionalProperties: false
+                              disposition:
+                                type: string
+                            required:
+                              - date and time received
+                              - diagnosis you disagree with
+                              - disposition
+                            additionalProperties: false
+                    required:
+                      - details about your request
+                    additionalProperties: false
+                required:
+                  - tool collection
+                  - tool name
+                  - this request
+                  - URLs of similar requests for web users
+                  - response metadata
+                  - response content
+                additionalProperties: false
   /api/orderNewTest:
     post:
       operationId: orderNewTest
@@ -1650,15 +1832,21 @@ components:
                   type: string
                 retesting:
                   type: string
+                approval:
+                  type: string
                 latency:
                   type: string
-                confirmation:
+                request confirmation:
+                  type: string
+                order confirmation:
                   type: string
               required:
                 - new testing
                 - retesting
+                - approval
                 - latency
-                - confirmation
+                - request confirmation
+                - order confirmation
               additionalProperties: false
           required:
             - what Kilotest does

package/openapi.yaml

@@ -2,7 +2,7 @@ openapi: 3.1.0
 info:
   title: Kilotest API
   description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and web standards, including HTML, CSS, ARIA, and WCAG). Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access. With this API you can request that a page be tested or retested, discover available reports, incrementally retrieve facts from a report: from what reports are available, to what issues were reported in one report, to what elements violated one issue, to what diagnoses rule engines gave for one violation.'
-  version: 61.0.0
+  version: 63.0.0
   license:
     name: MIT
     identifier: MIT
@@ -559,14 +559,14 @@ paths:
                                 how to get details about the element:
                                   type: object
                                   properties:
-                                    URL:
-                                      type: string
-                                    request method:
+                                    method:
                                       type: string
                                       const: GET
+                                    URL:
+                                      type: string
                                   required:
+                                    - method
                                     - URL
-                                    - request method
                                   additionalProperties: false
                                 web users can get details about the element at:
                                   type: string
@@ -709,6 +709,8 @@ paths:
                                 items:
                                   type: object
                                   properties:
+                                    identifier of the rule engine:
+                                      description: Copied from the report act (which); type not guaranteed.
                                     identifier of the violated rule:
                                       description: Copied from the report instance (ruleID); type not guaranteed.
                                     description of the violation:
@@ -717,11 +719,40 @@ paths:
                                       description: Copied from the report instance (ordinalSeverity); type not guaranteed.
                                     count of violations of the rule by the element:
                                       type: number
+                                    how to report that you disagree with this diagnosis:
+                                      type: object
+                                      properties:
+                                        method:
+                                          type: string
+                                          const: POST
+                                        URL:
+                                          type: string
+                                        request body:
+                                          type: object
+                                          properties:
+                                            engineID:
+                                              type: string
+                                            ruleID:
+                                              type: string
+                                            reason:
+                                              type: string
+                                          required:
+                                            - engineID
+                                            - ruleID
+                                            - reason
+                                          additionalProperties: false
+                                      required:
+                                        - method
+                                        - URL
+                                        - request body
+                                      additionalProperties: false
                                   required:
+                                    - identifier of the rule engine
                                     - identifier of the violated rule
                                     - description of the violation
                                     - severity of the violation on a 0-to-3 scale
                                     - count of violations of the rule by the element
+                                    - how to report that you disagree with this diagnosis
                                   additionalProperties: false
                               - type: object
                                 properties:
@@ -1179,6 +1210,153 @@ paths:
                   - response metadata
                   - response content
                 additionalProperties: false
+  /api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}:
+    post:
+      operationId: reportDisagreement
+      summary: Report disagreement with a diagnosis
+      parameters:
+        - in: path
+          name: issueID
+          schema:
+            type: string
+            description: 'Issue identifier (example: contrastPoor)'
+          required: true
+          description: 'Issue identifier (example: contrastPoor)'
+        - in: path
+          name: reportID
+          schema:
+            type: string
+            description: 'Identifier of the report (example: 260503T0432-x9z)'
+          required: true
+          description: 'Identifier of the report (example: 260503T0432-x9z)'
+        - in: path
+          name: catalogIndex
+          schema:
+            type: string
+            description: 'Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'
+          required: true
+          description: 'Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                engineID:
+                  type: string
+                  description: 'Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output'
+                ruleID:
+                  type: string
+                  description: 'Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead'
+                reason:
+                  type: string
+                  description: 20- to 1000-character explanation of why the diagnosis is disputed
+              required:
+                - engineID
+                - ruleID
+                - reason
+      responses:
+        '200':
+          description: Report disagreement with a diagnosis
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tool collection:
+                    $ref: '#/components/schemas/ToolsFacts'
+                  tool name:
+                    type: string
+                  this request:
+                    type: object
+                    properties:
+                      description:
+                        type: string
+                      method:
+                        type: string
+                        const: POST
+                      URL:
+                        type: string
+                      body:
+                        type: object
+                        properties:
+                          engineID:
+                            type: string
+                            description: 'Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output'
+                          ruleID:
+                            type: string
+                            description: 'Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead'
+                          reason:
+                            type: string
+                            description: 20- to 1000-character explanation of why the diagnosis is disputed
+                        required:
+                          - engineID
+                          - ruleID
+                          - reason
+                        additionalProperties: false
+                      closest ancestor request:
+                        anyOf:
+                          - $ref: '#/components/schemas/RequestReference'
+                          - type: 'null'
+                    required:
+                      - description
+                      - method
+                      - URL
+                      - body
+                      - closest ancestor request
+                    additionalProperties: false
+                  URLs of similar requests for web users:
+                    type: 'null'
+                  response metadata:
+                    $ref: '#/components/schemas/ResponseMetadata'
+                  response content:
+                    type: object
+                    properties:
+                      details about your request:
+                        anyOf:
+                          - type: object
+                            properties:
+                              error:
+                                type: string
+                            required:
+                              - error
+                            additionalProperties: false
+                          - type: object
+                            properties:
+                              date and time received:
+                                type: string
+                              diagnosis you disagree with:
+                                type: object
+                                properties:
+                                  issue identifier:
+                                    type: string
+                                  identifier of the rule engine:
+                                    type: string
+                                  identifier of the violated rule:
+                                    type: string
+                                required:
+                                  - issue identifier
+                                  - identifier of the rule engine
+                                  - identifier of the violated rule
+                                additionalProperties: false
+                              disposition:
+                                type: string
+                            required:
+                              - date and time received
+                              - diagnosis you disagree with
+                              - disposition
+                            additionalProperties: false
+                    required:
+                      - details about your request
+                    additionalProperties: false
+                required:
+                  - tool collection
+                  - tool name
+                  - this request
+                  - URLs of similar requests for web users
+                  - response metadata
+                  - response content
+                additionalProperties: false
   /api/orderNewTest:
     post:
       operationId: orderNewTest
@@ -1636,15 +1814,21 @@ components:
                   type: string
                 retesting:
                   type: string
+                approval:
+                  type: string
                 latency:
                   type: string
-                confirmation:
+                request confirmation:
+                  type: string
+                order confirmation:
                   type: string
               required:
                 - new testing
                 - retesting
+                - approval
                 - latency
-                - confirmation
+                - request confirmation
+                - order confirmation
               additionalProperties: false
           required:
             - what Kilotest does

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "61.0.0",
+  "version": "63.0.0",
   "description": "Ensemble testing for front-end web quality",
   "type": "module",
   "main": "index.ts",