@jrpool/kilotest 60.9.1 → 62.0.0

package/lhm.plugin.json

@@ -96,81 +96,48 @@
           "response content": {
             "type": "object",
             "properties": {
+              "how to get test results about a page": {
+                "type": "object",
+                "properties": {
+                  "method": {
+                    "type": "string",
+                    "const": "GET"
+                  },
+                  "URL": {
+                    "type": "string",
+                    "description": "URL template; replace {identifier} with the identifier of the latest report about the page."
+                  }
+                },
+                "required": [
+                  "method",
+                  "URL"
+                ],
+                "additionalProperties": false
+              },
+              "web users can get test results about a page at": {
+                "type": "string",
+                "description": "URL template; replace {identifier} with the identifier of the latest report about the page."
+              },
               "basics about all tested pages": {
                 "type": "array",
                 "items": {
                   "type": "object",
                   "properties": {
-                    "tested web page": {
-                      "type": "object",
-                      "properties": {
-                        "description": {
-                          "type": "string"
-                        },
-                        "URL": {
-                          "type": "string"
-                        }
-                      },
-                      "required": [
-                        "description",
-                        "URL"
-                      ],
-                      "additionalProperties": false
-                    },
-                    "number of reports about the page": {
-                      "type": "number"
-                    },
-                    "basics about the latest report": {
-                      "type": "object",
-                      "properties": {
-                        "identifier": {
-                          "type": "string",
-                          "description": "Identifier of the report (example: 260503T0432-x9z)."
-                        },
-                        "completion date and time": {
-                          "type": "string"
-                        },
-                        "days since the report was completed": {
-                          "type": [
-                            "number",
-                            "null"
-                          ]
-                        }
-                      },
-                      "required": [
-                        "identifier",
-                        "completion date and time",
-                        "days since the report was completed"
-                      ],
-                      "additionalProperties": false
-                    },
-                    "how to get details about the latest report": {
-                      "type": "object",
-                      "properties": {
-                        "method": {
-                          "type": "string",
-                          "const": "GET"
-                        },
-                        "URL": {
-                          "type": "string"
-                        }
-                      },
-                      "required": [
-                        "method",
-                        "URL"
-                      ],
-                      "additionalProperties": false
+                    "description": {
+                      "type": "string"
                     },
-                    "web users can get details about the latest report at": {
+                    "URL": {
                       "type": "string"
+                    },
+                    "identifier": {
+                      "type": "string",
+                      "description": "Identifier of the latest report about the page (example: 260503T0432-x9z)."
                     }
                   },
                   "required": [
-                    "tested web page",
-                    "number of reports about the page",
-                    "basics about the latest report",
-                    "how to get details about the latest report",
-                    "web users can get details about the latest report at"
+                    "description",
+                    "URL",
+                    "identifier"
                   ],
                   "additionalProperties": false
                 }
@@ -231,6 +198,8 @@
               }
             },
             "required": [
+              "how to get test results about a page",
+              "web users can get test results about a page at",
               "basics about all tested pages",
               "how to request that a page with no report be tested",
               "how a web user can request that the page be tested"
@@ -1272,17 +1241,17 @@
                         "how to get details about the element": {
                           "type": "object",
                           "properties": {
-                            "URL": {
-                              "type": "string"
-                            },
-                            "request method": {
+                            "method": {
                               "type": "string",
                               "const": "GET"
+                            },
+                            "URL": {
+                              "type": "string"
                             }
                           },
                           "required": [
-                            "URL",
-                            "request method"
+                            "method",
+                            "URL"
                           ],
                           "additionalProperties": false
                         },
@@ -3402,6 +3371,336 @@
         "taskSupport": "forbidden"
       }
     },
+    {
+      "name": "reportDisagreement",
+      "description": "Reports that you disagree with a diagnosis shown by the listDiagnoses tool; queued for manager review. Use this after listDiagnoses when you believe a rule violation was incorrectly diagnosed or classified into an issue, providing the engineID and ruleID shown in the diagnosis and your reason. Returns a confirmation with disposition details.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "issueID": {
+            "type": "string",
+            "description": "Issue identifier (example: contrastPoor)"
+          },
+          "reportID": {
+            "type": "string",
+            "description": "Identifier of the report (example: 260503T0432-x9z)"
+          },
+          "catalogIndex": {
+            "type": "string",
+            "description": "Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)"
+          },
+          "engineID": {
+            "type": "string",
+            "description": "Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output"
+          },
+          "ruleID": {
+            "type": "string",
+            "description": "Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead"
+          },
+          "reason": {
+            "type": "string",
+            "description": "20- to 1000-character explanation of why the diagnosis is disputed"
+          }
+        },
+        "required": [
+          "issueID",
+          "reportID",
+          "catalogIndex",
+          "engineID",
+          "ruleID",
+          "reason"
+        ],
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "outputSchema": {
+        "type": "object",
+        "properties": {
+          "tool collection": {
+            "$ref": "#/definitions/ToolsFacts"
+          },
+          "tool name": {
+            "type": "string"
+          },
+          "this request": {
+            "type": "object",
+            "properties": {
+              "description": {
+                "type": "string"
+              },
+              "method": {
+                "type": "string",
+                "const": "POST"
+              },
+              "URL": {
+                "type": "string"
+              },
+              "body": {
+                "type": "object",
+                "properties": {
+                  "engineID": {
+                    "type": "string",
+                    "description": "Identifier of the rule engine that produced the disputed diagnosis (example: axe), as shown in the listDiagnoses output"
+                  },
+                  "ruleID": {
+                    "type": "string",
+                    "description": "Identifier of the rule whose diagnosis is disputed, as shown in the listDiagnoses output (example: color-contrast); if that output shows null for the rule identifier, use its description of the violation instead"
+                  },
+                  "reason": {
+                    "type": "string",
+                    "description": "20- to 1000-character explanation of why the diagnosis is disputed"
+                  }
+                },
+                "required": [
+                  "engineID",
+                  "ruleID",
+                  "reason"
+                ],
+                "additionalProperties": false
+              },
+              "closest ancestor request": {
+                "anyOf": [
+                  {
+                    "$ref": "#/definitions/RequestReference"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "description",
+              "method",
+              "URL",
+              "body",
+              "closest ancestor request"
+            ],
+            "additionalProperties": false
+          },
+          "URLs of similar requests for web users": {
+            "type": "null"
+          },
+          "response metadata": {
+            "$ref": "#/definitions/ResponseMetadata"
+          },
+          "response content": {
+            "type": "object",
+            "properties": {
+              "details about your request": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "error": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "error"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "date and time received": {
+                        "type": "string"
+                      },
+                      "diagnosis you disagree with": {
+                        "type": "object",
+                        "properties": {
+                          "issue identifier": {
+                            "type": "string"
+                          },
+                          "identifier of the rule engine": {
+                            "type": "string"
+                          },
+                          "identifier of the violated rule": {
+                            "type": "string"
+                          }
+                        },
+                        "required": [
+                          "issue identifier",
+                          "identifier of the rule engine",
+                          "identifier of the violated rule"
+                        ],
+                        "additionalProperties": false
+                      },
+                      "disposition": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "date and time received",
+                      "diagnosis you disagree with",
+                      "disposition"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              }
+            },
+            "required": [
+              "details about your request"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "required": [
+          "tool collection",
+          "tool name",
+          "this request",
+          "URLs of similar requests for web users",
+          "response metadata",
+          "response content"
+        ],
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "additionalProperties": false,
+        "definitions": {
+          "ToolsFacts": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "description": {
+                "type": "object",
+                "properties": {
+                  "what Kilotest does": {
+                    "type": "string"
+                  },
+                  "how to retrieve findings": {
+                    "type": "object",
+                    "properties": {
+                      "level 1": {
+                        "type": "string"
+                      },
+                      "level 2": {
+                        "type": "string"
+                      },
+                      "level 3": {
+                        "type": "string"
+                      },
+                      "level 4": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "level 1",
+                      "level 2",
+                      "level 3",
+                      "level 4"
+                    ],
+                    "additionalProperties": false
+                  },
+                  "how to generate more findings": {
+                    "type": "object",
+                    "properties": {
+                      "new testing": {
+                        "type": "string"
+                      },
+                      "retesting": {
+                        "type": "string"
+                      },
+                      "latency": {
+                        "type": "string"
+                      },
+                      "confirmation": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "new testing",
+                      "retesting",
+                      "latency",
+                      "confirmation"
+                    ],
+                    "additionalProperties": false
+                  }
+                },
+                "required": [
+                  "what Kilotest does",
+                  "how to retrieve findings",
+                  "how to generate more findings"
+                ],
+                "additionalProperties": false,
+                "description": "Explanation of what Kilotest does and how to use its tools."
+              },
+              "URL": {
+                "type": "string",
+                "description": "URL of the Kilotest MCP server."
+              },
+              "web users can obtain similar functionalities at": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "name",
+              "description",
+              "URL",
+              "web users can obtain similar functionalities at"
+            ],
+            "additionalProperties": false
+          },
+          "RequestReference": {
+            "type": "object",
+            "properties": {
+              "tool name": {
+                "type": "string"
+              },
+              "description": {
+                "type": "string"
+              },
+              "method": {
+                "type": "string",
+                "enum": [
+                  "GET",
+                  "POST"
+                ]
+              },
+              "URL": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "tool name",
+              "description",
+              "method",
+              "URL"
+            ],
+            "additionalProperties": false
+          },
+          "ResponseMetadata": {
+            "type": "object",
+            "properties": {
+              "identifier": {
+                "type": "string",
+                "description": "Unique identifier of this response (timestamp and random suffix)."
+              },
+              "date and time": {
+                "type": "string",
+                "description": "UTC date and time when the response was generated, in ISO 8601 format."
+              }
+            },
+            "required": [
+              "identifier",
+              "date and time"
+            ],
+            "additionalProperties": false
+          }
+        }
+      },
+      "annotations": {
+        "title": "Report disagreement with a diagnosis",
+        "readOnlyHint": false,
+        "destructiveHint": false,
+        "idempotentHint": false,
+        "openWorldHint": false
+      },
+      "execution": {
+        "taskSupport": "forbidden"
+      }
+    },
     {
       "name": "orderNewTest",
       "description": "Orders testing of a page that has no report yet, with automated approval or rejection and immediate enqueueing if approved. Use this instead of requestNewTest when immediate testing is wanted; testing typically takes 2 to 4 minutes. Returns an order confirmation and disposition details, including, if approved, a report identifier (timestamp and job identifier) and a question about whether to wait for completion with the awaitTest tool.",
@@ -4423,5 +4722,5 @@
       }
     }
   ],
-  "version": "60.9.1"
+  "version": "62.0.0"
 }

package/llms-full.txt

@@ -74,7 +74,7 @@ Kilotest offers these tools:
 
 - `listPages`
   - method: `GET`
-  - purpose: list all tested pages, each with its latest report and its number of reports
+  - purpose: list all tested pages, each with the identifier of its latest report
   - path: `/api/listPages`
   - caveat: Cached results are likely to be stale and should not be used
 - `listIssues`
@@ -124,6 +124,19 @@ Kilotest offers these tools:
   - path: `/api/requestFeature`
   - payload properties
     - `feature`: description and motivation of the feature improvement or the new feature
+- `reportDisagreement`
+  - method: `POST`
+  - purpose: report disagreement with a diagnosis shown by `listDiagnoses` of how an element exhibited an issue
+  - path: `/api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}`
+  - parameters
+    - `issueID`: identifier of the issue
+    - `reportID`: identifier of the report, with YYMMDDTHHmm format for its initial segment (example: 260503T0432-x9z)
+    - `catalogIndex`: index of the catalog entry of the element
+  - source of parameters: response to a `listDiagnoses` request
+  - payload properties
+    - `engineID`: identifier of the rule engine that produced the disputed diagnosis, shown in the `listDiagnoses` response
+    - `ruleID`: identifier of the rule whose diagnosis is disputed, shown in the `listDiagnoses` response, or, if it is null, the description of the violation
+    - `reason`: 20- to 1000-character explanation of why the diagnosis is disputed
 - `orderNewTest`
   - method: `POST`
   - purpose: order immediate, automatically approved testing of one web page about which no report is available, instead of waiting for manual approval
@@ -163,7 +176,7 @@ When you request a test or a retest:
   - approval followed by the requested testing
   - rejection
 - how to confirm approval
-  - check test completion by calling `listPages` and looking for the page and the completion date of its latest report in the list
+  - check test completion by calling `listPages` and looking for the page in the list
   - you advise the user to check in approximately 1 day for the test results on the Kilotest website, starting at `https://kilotest.com/listPages.html`
 
 For immediate testing without waiting for manual approval, use `orderNewTest` or `orderRetest` instead of `requestNewTest` or `requestRetest`. Each enqueues the job right away and returns a report identifier along with a question about whether to wait for completion. To wait, call `awaitTest` with that identifier; it blocks until the report is ready, the job fails, or a maximum wait time elapses. To check later instead, call `listIssues` with the same identifier.

package/llms.txt

@@ -12,6 +12,7 @@ Kilotest integrates an ensemble of 12 rule engines, with about 1,300 rules, to t
 - Request testing of a web page: `/api/requestNewTest` (`POST`)
 - Request retesting of a web page: `/api/requestRetest/{reportID}` (`POST`)
 - Request adding or improving a feature: `/api/requestFeature` (`POST`)
+- Report disagreement with a diagnosis: `/api/reportDisagreement/{issueID}/{reportID}/{catalogIndex}` (`POST`)
 - Order immediate, automatically approved testing of a web page: `/api/orderNewTest` (`POST`)
 - Order immediate, automatically approved retesting of a web page: `/api/orderRetest/{reportID}` (`POST`)
 - Wait for a job ordered via orderNewTest or orderRetest to complete: `/api/awaitTest/{reportID}` (`POST`)

package/mcp.test.ts

@@ -37,10 +37,10 @@ test('mcpPath is /mcp', () => {
   assert.equal(mcpPath, '/mcp');
 });
 
-test('createMCPServer registers all 12 tools', () => {
+test('createMCPServer registers all 13 tools', () => {
   const server = createMCPServer();
   const toolNames = Object.keys((server as any)._registeredTools);
-  assert.equal(toolNames.length, 12);
+  assert.equal(toolNames.length, 13);
   assert.deepEqual(toolNames, [
     'getKilotestOverview',
     'listPages',
@@ -51,6 +51,7 @@ test('createMCPServer registers all 12 tools', () => {
     'requestNewTest',
     'requestRetest',
     'requestFeature',
+    'reportDisagreement',
     'orderNewTest',
     'orderRetest',
     'awaitTest'
@@ -197,6 +198,23 @@ test('requestFeature handler returns content and structuredContent', async () =>
   assert.equal(getToolCallCount('requestFeature'), countBefore + 1);
 });
 
+test('reportDisagreement handler returns content and structuredContent', async () => {
+  const server = createMCPServer();
+  const countBefore = getToolCallCount('reportDisagreement');
+  const result = await (server as any)._registeredTools.reportDisagreement.handler({
+    issueID: 'linkNoText',
+    reportID: '260101T0000-mix',
+    catalogIndex: '0',
+    engineID: 'axe',
+    ruleID: 'r11',
+    reason: 'The element does have an accessible name'
+  });
+  assert.ok(result.content);
+  assert.equal(result.content[0].type, 'text');
+  assert.ok(result.structuredContent);
+  assert.equal(getToolCallCount('reportDisagreement'), countBefore + 1);
+});
+
 test('orderNewTest handler returns content and structuredContent, and enqueues a job', async () => {
   const {jobsPath} = await import('./util.ts');
   const server = createMCPServer();
@@ -349,7 +367,7 @@ test('handleMCP responds to initialize with server info', async () => {
   }
 });
 
-test('handleMCP lists all 12 tools via tools/list', async () => {
+test('handleMCP lists all 13 tools via tools/list', async () => {
   const server = await startMCPServer();
   try {
     const port = server.address().port;
@@ -357,7 +375,7 @@ test('handleMCP lists all 12 tools via tools/list', async () => {
     assert.equal(res.statusCode, 200);
     const result = parseSSEResult(res.body);
     const toolNames = result.result.tools.map((t: any) => t.name);
-    assert.equal(toolNames.length, 12);
+    assert.equal(toolNames.length, 13);
     assert.deepEqual(toolNames, [
       'getKilotestOverview',
       'listPages',
@@ -368,6 +386,7 @@ test('handleMCP lists all 12 tools via tools/list', async () => {
       'requestNewTest',
       'requestRetest',
       'requestFeature',
+      'reportDisagreement',
       'orderNewTest',
       'orderRetest',
       'awaitTest'

package/mcp.ts

@@ -19,6 +19,7 @@ import * as requestFeatureAPI from './api/requestFeature.ts';
 import * as orderNewTestAPI from './api/orderNewTest.ts';
 import * as orderRetestAPI from './api/orderRetest.ts';
 import * as awaitTestAPI from './api/awaitTest.ts';
+import * as reportDisagreementAPI from './api/reportDisagreement.ts';
 import {version} from './api/version.ts';
 import {recordMetric} from './util.ts';
 
@@ -33,6 +34,7 @@ import {
   orderNewTestSchema,
   orderRetestSchema,
   awaitTestSchema,
+  reportDisagreementSchema,
   listPagesResponseSchema,
   listIssuesResponseSchema,
   listViolatorsResponseSchema,
@@ -43,7 +45,8 @@ import {
   requestFeatureResponseSchema,
   orderNewTestResponseSchema,
   orderRetestResponseSchema,
-  awaitTestResponseSchema
+  awaitTestResponseSchema,
+  reportDisagreementResponseSchema
 } from './api/schemas.ts';
 
 // CONSTANTS
@@ -257,6 +260,26 @@ export const createMCPServer = (): McpServer => {
       return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result};
     }
   );
+  server.registerTool(
+    'reportDisagreement',
+    {
+      description: 'Reports that you disagree with a diagnosis shown by the listDiagnoses tool; queued for manager review. Use this after listDiagnoses when you believe a rule violation was incorrectly diagnosed or classified into an issue, providing the engineID and ruleID shown in the diagnosis and your reason. Returns a confirmation with disposition details.',
+      inputSchema: reportDisagreementSchema,
+      outputSchema: reportDisagreementResponseSchema,
+      annotations: {
+        title: 'Report disagreement with a diagnosis',
+        readOnlyHint: false,
+        idempotentHint: false,
+        destructiveHint: false,
+        openWorldHint: false
+      }
+    },
+    async ({issueID, reportID, catalogIndex, engineID, ruleID, reason}) => {
+      const result = await reportDisagreementAPI.response([issueID, reportID, catalogIndex, engineID, ruleID, reason]);
+      await recordMetric('mcpToolCalls', 'reportDisagreement');
+      return {content: [{type: 'text', text: JSON.stringify(result)}], structuredContent: result};
+    }
+  );
   server.registerTool(
     'orderNewTest',
     {