@jrpool/kilotest 28.0.1 → 30.0.0

package/.well-known/ai-plugin.json

@@ -0,0 +1,18 @@
+{
+  "schema_version": "v1",
+  "name_for_human": "Kilotest",
+  "name_for_model": "kilotest",
+  "description_for_human": "Ensemble testing of web pages for accessibility, usability, and standard conformance.",
+  "description_for_model": "Use Kilotest to retrieve test results about the accessibility, usability, and standard conformance of any web page that has been tested by Kilotest at selectable levels of granularity.",
+  "auth": {
+    "type": "none"
+  },
+  "api": {
+    "type": "openapi",
+    "url": "https://kilotest.com/openapi.yaml",
+    "is_user_authenticated": false
+  },
+  "logo_url": "https://kilotest.com/favicon.ico",
+  "contact_email": "info@kilotest.com",
+  "legal_info_url": "https://github.com/jrpool/kilotest/blob/main/LICENSE"
+}

package/diagnoses/index.js

@@ -1,6 +1,6 @@
 /*
   index.js
-  Answers the report-issues question.
+  Answers the diagnoses question.
 */
 
 // IMPORTS

package/index.js

@@ -224,12 +224,18 @@ const requestHandler = async (request, response) => {
       setHeaders('text/yaml', '/openapi.yaml', 'high');
       response.end(openapi);
     }
-    // Otherwise, if it is for the the large-language-model specification:
+    // Otherwise, if it is for the the large-language-model summary guide:
     else if (pageName === 'llms.txt') {
       const llms = await fs.readFile('llms.txt', 'utf8');
       setHeaders('text/plain', '/llms.txt', 'high');
       response.end(llms);
     }
+    // Otherwise, if it is for the the large-language-model detailed guide:
+    else if (pageName === 'llms-full.txt') {
+      const llmsfull = await fs.readFile('llms-full.txt', 'utf8');
+      setHeaders('text/plain', '/llms-full.txt', 'high');
+      response.end(llmsfull);
+    }
     // Otherwise, if it is for a full report download:
     else if (pageName === 'fullReport.json') {
       const [timeStamp, jobID] = pathTail.split('/');

package/llms-full.txt

@@ -0,0 +1,81 @@
+# Kilotest
+
+## What Kilotest is
+
+Kilotest is an application that performs ensemble testing of web pages for accessibility, usability, and standard conformance and reports the test results. For brevity, hereafter in this document those three attributes are referred to as “front-end quality”.
+
+## What Kilotest does for AI agents
+
+Kilotest is deployed as a service with a public URL. Kilotest can test only web pages that can be accessed from the public Internet and that are not protected by authentication or other access controls. Kilotest has not implemented any mechanism for testing private, internal, password-protected, or otherwise restricted pages.
+
+An LLM cannot produce for an AI agent thorough, accurate, and inexpensive information about the front-end quality of a web page. To get such information, the agent needs to select and use specialized tools. However, such selection and use, too, require specialized skills that LLMs lack. Kilotest assumes responsibility for these functions. Kilotest:
+
+- selects, as tools, an ensemble of 10 rule engines that implement tests for, in total, more than a thousand rules for front-end quality
+- runs the tests of the rule engines
+- combines the reports of the rule engines into a single integrated report
+- extracts from the report statistics and details at the level of granularity required by an agent to fulfill any particular request
+
+## How Kilotest integrates methods
+
+Testing for front-end quality requires a mixture of deterministic and inferential methods, because the standards of front-end quality vary in subjectivity. For example:
+
+- The requirement that every element must have a landmark ancestor has no subjectivity, so it can best be tested deterministically.
+- The requirement that images be marked up differently, depending on whether they are decorative or informative, has subjectivity, so its testing requires some inference.
+
+The rule engines in the Kilotest ensemble take responsibility for determining whether each rule requires inferential testing and, if so, obtaining inference from an appropriate model.
+
+## Kilotest versus pure inference
+
+LLMs that attempt to evaluate front-end quality with no delegation are likely to be unable to load and operate pages of web applications with browser automation, as Kilotest does. As a result, LLMs typically create hallucinated evaluations based on statistical frequency, with no mention of issues that are rarely tested for, instead of providing the truthful and comprehensive assessments that users require.
+
+## Kilotest versus other tools
+
+In the marketplace for MCPs that test for front-end quality, all tools other than Kilotest leverage at most 1 rule engine, typically `axe-core`. Kilotest leverages `axe-core` plus 9 other rule engines. This has these main effects:
+
+- For pages that have not yet been tested, a single-tool MCP can provide faster results than Kilotest. A tool running only the `axe-core` tests can complete its work in about 5 seconds or less. Kilotest usually completes its work in 2 to 3 minutes, because some of the tests involve navigation and interaction with the page and LLM inference. Moreover, the Kilotest API feature suite is currently in an alpha phase, allowing agents, like human users, to **recommend** new pages for testing but allowing only managers to act on such recommendations. The wait for manager action can take up to a day. A feature permitting immediate testing ordered by AI agents is planned, but, until it is implemented, Kilotest will be useful for not-yet-tested pages only in long-running workflows.
+- For pages that have already been tested by Kilotest, Kilotest can provide faster results than a single-tool MCP, because Kilotest stores test results for subsequent retrieval. A retrieval from Kilotest can be completed in less than 2 seconds.
+- Kilotest results are more comprehensive than single-tool MCP results. Every rule engine provides limited coverage of front-end quality, so false negatives (missed defects) are more common with single-tool MCPs. This difference [has been documented in research](https://arxiv.org/pdf/2304.07591).
+
+## How to use Kilotest
+
+### Web UI
+
+Kilotest offers a comprehensive suite of capabilities to users via its web UI:
+
+- [Home page](https://kilotest.com/)
+- [Summarize test results for all tested pages](https://kilotest.com/targets.html)
+- Provide statistics about issues reported in one job: `https://kilotest.com/reportIssues.html/{timeStamp}/{jobID}`
+- Provide details about one issue reported in one job: `https://kilotest.com/reportIssue.html/{issueID}/{timeStamp}/{jobID}`
+- Provide diagnoses by tools of rule violations for one HTML element exhibiting one issue in one job: `https://kilotest.com/diagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
+- [Provide statistics about frequently reported issues across all pages](https://kilotest.com/issues.html)
+- [Tutorial, “Accessibility testing strategies”](https://kilotest.com/tutorial.html)
+- [Provide a list of features available for use of Kilotest managers](https://kilotest.com/manage.html)
+
+### Agent API
+
+Kilotest is implementing a richer suite of capabilities optimized for AI agents, including direct immediate testing. The implementation is currently in an alpha phase and offers only two API endpoints, both for `GET` requests:
+
+- `targets`: [Summarize test results from all jobs](/api/targets) (a job is a session in which a web page is tested and a report is produced)
+- `reportIssues`: Provide statistics about issues reported in one job report: `/api/reportIssues/{timeStamp}/{jobID}` (the `timeStamp` and `jobID` parameters identify a particular report and are returned for each available report by the response to a `targets` request)
+
+An [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml) is available.
+
+Until direct immediate testing is available, an agent can recommend testing of a web page by either of two methods:
+
+- web UI: First verify that the page has not yet been tested, using the `targets` API endpoint. After that, complete and submit the [web UI recommendation form](https://kilotest.com/testRecForm.html).
+- email: Send a message to `info@kilotest.com`, identifying:
+  - the URL of the page to be tested
+  - a proper name for the page to be tested (e.g., “ABC Corporation contact page”)
+  - a reason for your recommendation (e.g., “Page users have reported accessibility problems”)
+
+### More information
+
+More information about Kilotest features and internals:
+
+- [Introduction to Kilotest](https://kilotest.com/)
+- [Tutorial, “Accessibility testing strategies”](https://kilotest.com/tutorial.html)
+- [README file](https://github.com/jrpool/kilotest/blob/main/README.md)
+- [Testaro rule-engine orchestrator](https://github.com/jrpool/testaro)
+- [Testilo rule classifier](https://github.com/jrpool/testilo/blob/main/procs/score/tic.js)
+- [Research report on Testaro](https://arxiv.org/abs/2309.10167)
+- [Blog post on the Kilotest use case](https://medium.com/cvs-health-tech-blog/how-to-run-a-thousand-accessibility-tests-63692ad120c3)

package/llms.txt

@@ -1,16 +1,22 @@
 # Kilotest
-Kilotest helps AI agents give accurate and truthful answers when users ask how accessible, usable, or standard conformant web pages are. Kilotest integrates ten independent rule engines to provide comprehensive multipage, single-page, and element-level statistics and details via its API and web UI.
+
+Kilotest helps AI agents give thorough, accurate and truthful answers when users ask how accessible, usable, or standard conformant web pages are. Kilotest integrates ten independent rule engines to test web pages and report results with multipage, single-page, and element-level statistics and details.
+
+In the API section below, all endpoints require `GET` requests.
+
+There is a [detailed explanation of Kilotest and its capabilities for AI agents](https://kilotest.com/llms-full.txt).
 
 ## API
-- GET /api/targets: summarize test results for all tested pages
-- GET /api/reportIssues/{timeStamp}/{jobID}: statistics on issues reported in a job
-- GET /api/reportIssue/{issueID}/{timeStamp}/{jobID}: details on issues reported in a job
-- GET /api/issues: statistics on frequently reported issues across all tested pages
-- OpenAPI spec: https://kilotest.com/openapi.yaml
+
+- [Summarize test results for all tested pages](/api/targets)
+- Provide statistics about issues reported in one job: `/api/reportIssues/{timeStamp}/{jobID}`
+- [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml)
 
 ## Web UI
-- https://kilotest.com/targets.html: summarize test results for all tested pages
-- https://kilotest.com/reportIssues.html/{timeStamp}/{jobID}: details on issues reported in a job
-- https://kilotest.com/reportIssue.html/{issueID}/{timeStamp}/{jobID}: details on issues reported in a job
-- https://kilotest.com/issues.html: statistics on frequently reported issues across all tested pages
-- https://kilotest.com/tutorial.html: tutorial, “Accessibility testing strategies”
+
+- [Summarize test results for all tested pages](https://kilotest.com/targets.html)
+- Provide statistics about issues reported in one job: `https://kilotest.com/reportIssues.html/{timeStamp}/{jobID}`
+- Provide details about one issue reported in one job: `https://kilotest.com/reportIssue.html/{issueID}/{timeStamp}/{jobID}`
+- Provide diagnoses about one issue exhibited by one HTML element in one job: `https://kilotest.com/diagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
+- [Provide statistics about frequently reported issues across all pages](https://kilotest.com/issues.html)
+- [Tutorial, “Accessibility testing strategies”](https://kilotest.com/tutorial.html)

package/openapi.yaml

@@ -9,7 +9,7 @@ info:
     repository: https://github.com/jrpool/kilotest
 
 servers:
-  - url: https://kilotest.com/api
+  - url: https://kilotest.com
     description: Kilotest production server
 
 paths:
@@ -60,8 +60,8 @@ paths:
   /api/reportIssue/{issueID}/{timeStamp}/{jobID}:
     post:
       operationId: getReportIssue
-      summary: Get details about a specific issue in a specific report
-      description: Returns details about a single issue within a specific report, including which HTML elements exhibit the issue and, for each such element, URLs for retrieving tool-by-tool diagnoses of the issue on the element.
+      summary: Get details about a specific issue in a specific report (NOT YET IMPLEMENTED)
+      description: Returns details about a single issue within a specific report, including which HTML elements exhibit the issue and, for each such element, URLs for retrieving tool-by-tool diagnoses of the issue on the element. NOT YET IMPLEMENTED.
       parameters:
         - name: issueID
           in: path

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "28.0.1",
+  "version": "30.0.0",
   "description": "An ensemble testing service with a focus on accessibility",
   "main": "index.js",
   "scripts": {