@jrpool/kilotest 46.0.11 → 46.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +22 -1
- package/.github/workflows/ci.yml +1 -1
- package/.github/workflows/publish-mcp.yml +1 -1
- package/README.md +16 -8
- package/api/requestFeature.test.js +6 -2
- package/api/requestRetest.test.js +6 -2
- package/api/requestTest.test.js +6 -2
- package/api/util.js +1 -1
- package/capability.md +51 -0
- package/checkinfo.js +146 -0
- package/docs/AI-TOOLS.md +64 -151
- package/docs/SERVICE.md +1 -1
- package/index.html +4 -3
- package/index.js +27 -19
- package/index.test.js +41 -17
- package/lhm.plugin.json +3218 -0
- package/llms-full.txt +7 -7
- package/llms.txt +2 -1
- package/openapi.yaml +1 -1
- package/package.json +5 -4
- package/scripts/generate-openapi.js +1 -1
- package/smokeTest.js +28 -25
- package/util.test.js +2 -2
- package/web/listReports/listReports.test.js +1 -1
- package/web/reannotate/reannotate.test.js +2 -2
- package/web/requestTest/requestTest.test.js +3 -3
- package/web/rewindReportsForm/rewindReportsForm.test.js +1 -1
- package/web/tutorial/index.html +3 -3
- package/web/tutorial/tutorial.test.js +4 -4
package/docs/AI-TOOLS.md
CHANGED
|
@@ -4,56 +4,71 @@
|
|
|
4
4
|
|
|
5
5
|
Until 2026 Kilotest was intended, and implemented, as a web application performing a service for human users.
|
|
6
6
|
|
|
7
|
-
Beginning in May 2026, it [became evident](https://github.com/jrpool/kilotest/issues/2) that Kilotest could
|
|
7
|
+
Beginning in May 2026, it [became evident](https://github.com/jrpool/kilotest/issues/2) that Kilotest could operate as a set of tools for language models to allow them to improve their answers to questions about the front-end quality (accessibility, usability, and standards conformity) of specific web pages.
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
A decision was made to **make Kilotest discoverable, usable, and, where appropriate, used as a provider of tools for language models**.
|
|
10
10
|
|
|
11
11
|
## Terms
|
|
12
12
|
|
|
13
|
-
- **Language model**: A model (e.g., Claude Haiku 4.5, Kimi K2.
|
|
13
|
+
- **Language model**: A model (e.g., Claude Haiku 4.5, Kimi K2.7, GPT-5.4, Gemini 3.1 Pro) that can consume and generate text, images, or other content.
|
|
14
14
|
- **AI platform**: A platform (e.g., Claude Desktop, Perplexity, ChatGPT, Gemini 3.5 Flash) that gives human users access to the services of language models and connects language models to productivity resources.
|
|
15
15
|
- **tool**: A specialized productivity resource providing a capability that a language model needs but does not have. In the OpenAPI specification, the term _operation_ is used to mean tool.
|
|
16
16
|
- **connector**: A service that allows AI platforms to enable their language models to discover, evaluate, and use tools.
|
|
17
17
|
|
|
18
|
-
## Internal
|
|
18
|
+
## Internal features
|
|
19
19
|
|
|
20
|
-
The internal
|
|
20
|
+
The internal features that make Kilotest a collection of tools for language models are:
|
|
21
21
|
|
|
22
|
-
- An API,
|
|
23
|
-
|
|
24
|
-
- Additions to `util.js`.
|
|
25
|
-
- An `api` directory containing:
|
|
26
|
-
- Eight tool modules.
|
|
27
|
-
- A `util.js` module with shared functions used by tool modules.
|
|
28
|
-
- A `routes.js` file.
|
|
29
|
-
- A `schemas.js` file.
|
|
30
|
-
- A `testSuite.js` module for testing the API.
|
|
22
|
+
- An API, with specific functional and utility modules in the `api` directory.
|
|
23
|
+
- Tests of the functionalities of the API.
|
|
31
24
|
- A [`JSON-LD`](https://json-ld.org/) script in the `index.html` file, providing structured data about the Kilotest API.
|
|
32
|
-
-
|
|
25
|
+
- Environment variables in the `env.example` file.
|
|
33
26
|
- An `llms.txt` file and an `llms-full.txt` file, documenting the use of Kilotest by language models, conforming to the [llms-txt](https://llmstxt.org/) specification.
|
|
34
|
-
-
|
|
27
|
+
- A generated `openapi.yaml` file, documenting the Kilotest API, conforming to the [OpenAPI specification](https://spec.openapis.org/oas/v3.1.0).
|
|
35
28
|
- An `mcp.js` file, providing an MCP server for Kilotest.
|
|
29
|
+
- A `server.json` file, conforming to the [MCP server schema](https://modelcontextprotocol.io/), and a GitHub Actions workflow (`publish-mcp.yml`) that authenticates via GitHub OIDC and publishes that file to the official MCP Registry whenever `server.json` or `api/version.js` changes on the `main` branch.
|
|
30
|
+
- A `capability.md` file with a URL, summarizing what Kilotest tools can do.
|
|
31
|
+
- `mcp`, `mcp-server`, and `modelcontextprotocol` keywords in `package.json`, supporting discovery of the `@jrpool/kilotest` npm package by tools and humans searching the npm registry for MCP servers.
|
|
32
|
+
- An `lhm.plugin.json` file, conforming to the [LobeHub plugin manifest format](https://market.lobehub.com/s/publish-mcp/references/manifest), declaring Kilotest as a plugin in the [LobeHub MCP Market](https://lobehub.com/mcp). Publishing and updating are non-automated (`lhm plugin publish`/`update`), because LobeHub requires browser-based OAuth and does not support machine-to-machine publishing.
|
|
36
33
|
- A `sitemap.xml` file.
|
|
37
|
-
-
|
|
34
|
+
- Documentation in the `README.md` file.
|
|
38
35
|
- This `AI-TOOLS.md` file.
|
|
39
36
|
- A [tutorial on authorizing language models to use Kilotest](https://kilotest.com/qai/).
|
|
40
37
|
|
|
41
|
-
## External
|
|
38
|
+
## External features
|
|
42
39
|
|
|
43
|
-
The external
|
|
40
|
+
The external features that support the use of Kilotest as a collection of AI tools are:
|
|
44
41
|
|
|
45
42
|
- A [pull request](https://github.com/public-apis/public-apis/pull/6346/changes) to add Kilotest to the list of public APIs in the `public-apis` repository.
|
|
46
43
|
- An [issue](https://github.com/APIs-guru/openapi-directory/issues/2677) to add Kilotest to `openapi-directory`.
|
|
47
44
|
- A [pull request](https://github.com/w3c/wai-evaluation-tools-list/pull/1153) to add Kilotest to the WAI evaluation tools list.
|
|
45
|
+
- Registration of Kilotest as an active server in the [official MCP Registry](https://registry.modelcontextprotocol.io/v0/servers?search=kilotest) (`io.github.jrpool/kilotest`), maintained by the [Model Context Protocol](https://modelcontextprotocol.io/) project. The [PulseMCP](https://www.pulsemcp.com/servers) directory and the VS Code MCP gallery both crawl this registry, so Kilotest is discoverable through them with no separate submission.
|
|
48
46
|
- Registration of Kilotest with the [Smithery](https://smithery.ai/servers/pool/kilotest) MCP server registry.
|
|
49
47
|
- Registration of Kilotest with the [Glama](https://glama.ai/mcp/connectors/com.kilotest/kilotest) MCP server registry.
|
|
48
|
+
- Registration of Kilotest with [Awesome MCP Servers](https://www.tensorblock.co/mcp/servers/github-jrpool-kilotest-f518ba36).
|
|
50
49
|
- Registration of Kilotest with the [RapidAPI](https://rapidapi.com/jrpool/api/kilotest/playground/apiendpoint_0f03577a-ff9a-472a-a0ed-533bd198981a) Hub.
|
|
50
|
+
- Registration of Kilotest with the [LobeHub MCP Market](https://lobehub.com/mcp) (identifier: `jrpool-kilotest`), enabled with an `lhm.plugin.json` file.
|
|
51
51
|
- Deployment of an MCP server in HTTP mode on the Kilotest service host.
|
|
52
52
|
- Configuration of Claude Desktop on the local development host and the `claude.ai` web application to connect Claude Desktop models to the Kilotest MCP server. The configuration was performed in the UI of each platform with the addition of Kilotest as a _connector_. The user used the `Customize/Connectors/Add connector/Add custom connector` interface, providing these data before activating the `Add` button:
|
|
53
53
|
|
|
54
54
|
- Name: Kilotest
|
|
55
55
|
- Remote MCP server URL: `https://kilotest.com/mcp`
|
|
56
56
|
|
|
57
|
+
- MCP-related topics added to the [repository details](https://github.com/jrpool/kilotest).
|
|
58
|
+
- An [MCP Registry badge](https://registry.modelcontextprotocol.io/v0/servers?search=kilotest) added to the `README.md` file.
|
|
59
|
+
- A link to the MCP server on the `README.md` page of the [`testaro-issues`](https://github.com/jrpool/testaro-issues) repository.
|
|
60
|
+
- Registration of Kilotest and its sitemap at [Google Search Console](https://search.google.com/search-console?resource_id=sc-domain:kilotest.com) and [Bing Webmaster Tools](https://www.bing.com/webmasters/home?siteUrl=https://kilotest.com/).
|
|
61
|
+
|
|
62
|
+
## Required maintenance
|
|
63
|
+
|
|
64
|
+
Whenever the `/api/version.js` file is updated:
|
|
65
|
+
|
|
66
|
+
- Update the `server.json` file (version field).
|
|
67
|
+
- Update the `lhm.plugin.json` file (version field).
|
|
68
|
+
- Regenerate `openapi.yaml` (`npm run generate:openapi`).
|
|
69
|
+
- Create a pull request and merge to `main` (triggers `publish-mcp.yml` automatically for the MCP Registry).
|
|
70
|
+
- Execute `lhm plugin update --dir .` (for LobeHub).
|
|
71
|
+
|
|
57
72
|
## Use cases
|
|
58
73
|
|
|
59
74
|
The rationale for Kilotest as a collection of tools for language models is set forth in the `llms-full.txt` file and is not repeated here.
|
|
@@ -71,148 +86,46 @@ Among these use cases, case 3 would make it feasible for the user to tell an AI
|
|
|
71
86
|
|
|
72
87
|
Use cases 1, 2, 4, 5, and 6 exemplify a widespread expectation and demand for AI platform capability. The commonality is: “I have a question; answer it.” If Kilotest can be employed as an expert for AI platforms in relevant cases, platforms will be more successful in satisfying that demand. At present this is a difficult problem because of platform limitations and a lack of standardization.
|
|
73
88
|
|
|
74
|
-
##
|
|
75
|
-
|
|
76
|
-
Connector and tool discovery and utilization have been only partly standardized. Major differences in protocols exist among model and platform families. Therefore, small testable increments of improvement in the connector and tool functionality of Kilotest can best be defined by model and platform family and by use-case characteristics.
|
|
77
|
-
|
|
78
|
-
One benefit of such incrementalism is that, after the first increment succeeds, it becomes possible to make and test external changes publicizing the fact that a particular platform-model combination delivers unprecedentedly inexpensive, comprehensive, and truthful answers in a particular class of use cases to questions about front-end web quality.
|
|
79
|
-
|
|
80
|
-
Another benefit is that subsequent increments can be defined incrementally rather than in advance. Lessons learned from the work on each increment can inform the choice of what to work on next.
|
|
81
|
-
|
|
82
|
-
## Implementation history
|
|
83
|
-
|
|
84
|
-
The Kilotest project has been attempting to solve the just-described problem. A chronology of this work follows.
|
|
85
|
-
|
|
86
|
-
### Increment 1
|
|
87
|
-
|
|
88
|
-
In the first increment, the objective was to make Anthropic Claude models use Kilotest to help them answer questions from developers using the Claude Desktop application for use case 3, where the code in question is already deployed as a public web page.
|
|
89
|
-
|
|
90
|
-
A locally installed MCP server, `@ivotoby/openapi-mcp-server`, was used. It turned out to be incompatible with Claude Desktop, so even though connections to Kilotest were made and calls to its tools were submitted the responses did not arrive.
|
|
91
|
-
|
|
92
|
-
The model behavior motivated an architectural change in the MCP server and improvements in the naming and description of the Kilotest tools before the next increment.
|
|
93
|
-
|
|
94
|
-
### Increment 2
|
|
95
|
-
|
|
96
|
-
Increment 2 repeated the exercise of increment 1 with the `claude.ai` web application instead of Claude Desktop, and with the improvements indicated by increment 1 made.
|
|
97
|
-
|
|
98
|
-
With Kilotest added as a custom connector, the user asked the platform, “I want to know whether the home page of the nonprofit organization named ‘Open Secrets’ has quality defects, such as with accissibility [_sic_] and usability.”
|
|
99
|
-
|
|
100
|
-
It was discovered that Claude Haiku 4.5 with Extended capability used Kilotest tools, but Claude Haiku 4.5 without Extended capability did not.
|
|
101
|
-
|
|
102
|
-
When Kilotest was used, it was used appropriately for the task, but the planning was suboptimal, because the model initially stated an intention to recommend the web page for testing instead of first checking for an existing available report on it.
|
|
103
|
-
|
|
104
|
-
The output to the user was a mixture of Kilotest results with knowledge and judgments of the model. Kilotest results were presented with some selective pruning, interpretation, and advice on what to do.
|
|
105
|
-
|
|
106
|
-
The trial in increment 2 suggested some opportunities for further improvements:
|
|
107
|
-
|
|
108
|
-
- Clearer instructions about the sequencing of tool calls
|
|
109
|
-
- Addition of instructions to attribute the interpretive additions by the model to the model or its other sources rather than to Kilotest.
|
|
110
|
-
|
|
111
|
-
### Increment 3
|
|
112
|
-
|
|
113
|
-
After the above-described improvements were made, a similar exercise, with the home page of a company named “CMT Services”, was performed again in Claude Desktop. In this case Claude Haiku 4.5 succeeded in recognizing the relevance of Kilotest and used its tools, though imperfectly. Instead of using Kilotest to check for an existing report on the page, the model searched for it in a cached file with:
|
|
114
|
-
|
|
115
|
-
```bash
|
|
116
|
-
Bash
|
|
117
|
-
jq '.[] | .tested_web_page.description' /Users/pool/.claude/projects/-Users-pool-Documents/18b3821c-b60d-4f63-bf3f-542830ed3bc4/tool-results/toolu_012qMKW66d88vWjp8JKj5qNn.json 2>/dev/null | grep -i "cmt" || echo "CMT Services not found in tested pages"
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
Claude Haiku then incorrectly concluded: `CMT Services hasn't been tested yet. Let me search for their website so I can recommend it for quality testing.` The model asked the user for the URL and said:
|
|
121
|
-
|
|
122
|
-
```text
|
|
123
|
-
Once you provide the URL, I'll:
|
|
124
|
-
|
|
125
|
-
Recommend their home page for quality testing with Kilotest
|
|
126
|
-
Retrieve the quality report showing accessibility and usability defects
|
|
127
|
-
Provide you with a detailed analysis of any issues found
|
|
128
|
-
```
|
|
129
|
-
|
|
130
|
-
The user provided the URL, and Claude Haiku then submitted a recommendation to Kilotest to test the page. The submission was successful and, as expected, yielded an error message: `{"error":"A report about the page is already available"}`.
|
|
131
|
-
|
|
132
|
-
The model drew the correct conclusion from this and (contradicting itself) announced: `Great news—there's already a quality report for CMT Services! Let me retrieve it for you.` The model did so and gave the user an appropriate rendition of the results, modulo attribution. The results were introduced with `The Kilotest analysis found 51 quality defects on the CMT Services home page, affecting 163 HTML elements. Here's a breakdown:`. Some mildly interpretive additions, such as calling “high-priority” issues “critical”, were made without attribution, making it easy for a user to infer that they were made by Kilotest tools rather than the model.
|
|
133
|
-
|
|
134
|
-
The same trial was repeated, with a different website, using Claude Sonnet 4.6 with Low effort. This model immediately checked for and found an already available report and used it in the production of output to the user. The output was somewhat less interpretive than that of Claude Haiku.
|
|
135
|
-
|
|
136
|
-
Increment 3 showed that, after the naming, instructions, and connector configuration were improved, the relatively inexpensive Claude models use Kilotest where appropriate.
|
|
137
|
-
|
|
138
|
-
The misbehavior of Claude Haiku suggested that further improvements in the instructions may improve the utilization of Kilotest, namely:
|
|
139
|
-
|
|
140
|
-
- a warning not to use cached lists of available reports
|
|
141
|
-
- stronger advice to avoid implicitly attributing interpretations, judgments, and advice to Kilotest.
|
|
142
|
-
|
|
143
|
-
### Increment 4
|
|
144
|
-
|
|
145
|
-
The first trial in which the user asked about a page **without** an available report repeated the conditions of Increment 3, except for the web page in question and the model. The model here was Claude Haiku in Extended mode, an option available with `claude.ai` but not Claude Desktop.
|
|
146
|
-
|
|
147
|
-
The user asked: “I want to know whether the home page of Milgard Windows and Doors, at `www.milgard.com`, has quality defects, such as with accessibility and usability.”
|
|
148
|
-
|
|
149
|
-
The model immediately:
|
|
150
|
-
|
|
151
|
-
- found Kilotest
|
|
152
|
-
- checked for an available report about the page
|
|
153
|
-
- found no report
|
|
154
|
-
- submitted a recommendation to test the page
|
|
155
|
-
|
|
156
|
-
The recommendation was successfully received by the Kilotest service.
|
|
157
|
-
|
|
158
|
-
During the workflow the model kept the user informed. After the last step, the model summarized the type of report that the user could expect and advised the user to check directly with the Kilotest UI at `https://kilotest.com/targets` in 24 to 48 hours for the results.
|
|
159
|
-
|
|
160
|
-
The behavior of Claude Haiku in Extended mode was nearly perfect. The naming of the page in the recommendation was “Milgard Windows and Doors home page”, whereas “Milgard Windows and Doors” would have followed the existing naming pattern. This deviation suggests adding a specific instruction to name home pages without anything more than the name of the organization.
|
|
161
|
-
|
|
162
|
-
The most notable fact was that Claude Haiku converted a long workflow to a short one by ending the output with a recommendation to the user to get the test results by self-service. The instructions to models presumed that the user and the model would jointly decide to maintain their relationship until the testing is completed and the model would then interpret the results for the user. Instead, Claude Haiku had enough imagination to suggest a viable workflow not even hinted at by the instructions.
|
|
163
|
-
|
|
164
|
-
This result suggested that the instructions should be revised to describe, as a valid option, a short workflow ending with a recommendation to the user to get the test results by self-service after the model submits a testing recommendation.
|
|
165
|
-
|
|
166
|
-
### Increment 5
|
|
167
|
-
|
|
168
|
-
In the next increment the AI platform was changed to [Perplexity Pro](https://www.perplexity.ai/) on the web. With the platform UI, two configuration steps were taken:
|
|
169
|
-
|
|
170
|
-
1. Kilotest was added as a custom connector, and the list of connectors thereafter included Kilotest.
|
|
171
|
-
1. In the input section of the chat interface, with the “Add files or tools” button (visually `+`), Kilotest was added as a connector. Thereafter, a “Kilotest” button appeared in the input section to confirm the availability of that connector.
|
|
172
|
-
|
|
173
|
-
The first model used was GPT-5.4.
|
|
174
|
-
|
|
175
|
-
The prompt was: “I want to know whether the home page of the nonprofit organization named ‘Center for Democracy and Technology’ has quality defects, such as with accessibility and usability.”
|
|
176
|
-
|
|
177
|
-
GPT-5.4 responded by correctly using two Kilotest tools, first to check for an available report and then to inspect the results of the report about the specified web page. The model then appropriately itemized and summarized findings from the report, without any substantial interpretive additions.
|
|
178
|
-
|
|
179
|
-
The model was then changed to Sonar 2, created by Perplexity itself.
|
|
180
|
-
|
|
181
|
-
The results were similar, with one exception: The answer from Sonar 2 included advice and judgments (“most urgent fixes”, “especially notable”), without any attribution, so a user might wrongly infer that these were generated by Kilotest tools.
|
|
182
|
-
|
|
183
|
-
For both models, the same exercise was also attempted with the second configuration omitted, i.e. ••without** Kilotest being a named connector in the input section of the chat interface. In both cases, the models created their own unaided assessments and failed to mention or use the Kilotest tools.
|
|
184
|
-
|
|
185
|
-
The [Perplexity documentation](https://www.perplexity.ai/help-center/en/articles/13915507-adding-custom-remote-connectors) states that as of now the installed Perplexity application for macOS does not support remote connectors.
|
|
89
|
+
## Future work
|
|
186
90
|
|
|
187
|
-
|
|
91
|
+
To-dos recommended by Claude Sonnet 5 Medium on Devin:
|
|
188
92
|
|
|
189
|
-
|
|
93
|
+
- **No content/backlink outreach.** Registrations are passive listings; there's no blog post, Show HN, or subreddit post (r/modelcontextprotocol, r/accessibility, r/webdev) building external backlinks and social-discovery signal, which the doc's own "difficult problem…lack of standardization" framing suggests is exactly the gap left for the **human**-driven parts of discoverability.
|
|
190
94
|
|
|
191
|
-
|
|
95
|
+
- **Other directories not yet targeted.** The [mcp.so](https://mcp.so/submit) directory offers a free tier but steers submitters toward a $39 paid listing; deferred as a commercial decision. The Docker MCP Catalog is not a fit because it requires stdio transport and a Dockerfile, and Kilotest is streamable-http.
|
|
192
96
|
|
|
193
|
-
|
|
97
|
+
- **Client-onboarding friction isn't documented.** Some MCP clients need the `mcp-remote` npx bridge to use a remote streamable-HTTP server like Kilotest's. Ready-made Claude Desktop/Cursor/Windsurf config instructions added to QAI would convert "discovered" into "used" faster, since case 3 of your use cases assumes a technically savvy discoverer but not all users are.
|
|
194
98
|
|
|
195
|
-
|
|
99
|
+
Other future work:
|
|
196
100
|
|
|
197
|
-
|
|
101
|
+
- Add to QAI, or replace much of the current QAI content with, an [Add Custom Connector link](https://claude.ai/customize/connectors?modal=add-custom-connector&connectorName=Kilotest&connectorUrl=https%3A%2F%2Fkilotest.com%2Fmcp).
|
|
102
|
+
- Consider subscribing to a `Claude.ai` Team plan and submitting Kilotest to the [Claude Connectors Directory](https://claude.com/docs/connectors/building/submission). Details provided by GLM-5.2 High: “If accepted, Kilotest would get a permanent listing URL like `https://claude.ai/directory/connectors/kilotest` with a Connect button, and would appear in Claude’s in-product browse/search and ‘Suggested Connectors’. This is the highest-value listing for your use cases, because it puts Kilotest directly in front of Claude users who have not explicitly searched for it.”
|
|
103
|
+
- Recommended by GLM-5.2 High: Submit a Kilotest-appropriate request to one or more AI platforms and discover whether the model discovers Kilotest and, if so, either uses it or offers to use it in the creation of a response.
|
|
104
|
+
- Recommended by ChatGPT: A general capability manifest that says, in one machine-readable place: Here are the capabilities Kilotest provides; here are all the interfaces through which you can invoke them; here are their requirements and effects. For example:
|
|
198
105
|
|
|
199
|
-
|
|
106
|
+
```markdown
|
|
107
|
+
Capability: web-page-quality-audit
|
|
200
108
|
|
|
201
|
-
|
|
109
|
+
Purpose:
|
|
110
|
+
Test a public web page for accessibility,
|
|
111
|
+
usability, HTML and web-standards problems.
|
|
202
112
|
|
|
203
|
-
|
|
113
|
+
Interfaces:
|
|
114
|
+
MCP: https://kilotest.com/mcp
|
|
115
|
+
OpenAPI: https://kilotest.com/openapi.yaml
|
|
116
|
+
Web UI: https://kilotest.com/
|
|
204
117
|
|
|
205
|
-
|
|
118
|
+
Input:
|
|
119
|
+
Public URL
|
|
206
120
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
-
|
|
210
|
-
- listDiagnoses
|
|
211
|
-
- getReport
|
|
212
|
-
- requestRetest
|
|
213
|
-
- requestTest
|
|
214
|
-
- requestFeature
|
|
121
|
+
Output:
|
|
122
|
+
Structured findings, affected elements,
|
|
123
|
+
rule-engine diagnoses, WCAG mappings
|
|
215
124
|
|
|
216
|
-
|
|
125
|
+
Side effects:
|
|
126
|
+
Reads the specified public web page.
|
|
127
|
+
Does not modify the target site.
|
|
217
128
|
|
|
218
|
-
|
|
129
|
+
Cost:
|
|
130
|
+
...
|
|
131
|
+
```
|
package/docs/SERVICE.md
CHANGED
|
@@ -199,7 +199,7 @@ kilotest.com {
|
|
|
199
199
|
# Specify the only paths of forwardable requests.
|
|
200
200
|
@allowedGET {
|
|
201
201
|
method GET
|
|
202
|
-
path /mcp / /index.html /robots.txt /openapi.yaml /openapi.json /swagger.yaml /swagger.json /api-docs /llms.txt /llms-full.txt *.html* /fullReport.json/* /api/* /tutorial/images/* /favicon.* /style.css /sitemap.xml
|
|
202
|
+
path /mcp / /index.html /robots.txt /openapi.yaml /openapi.json /swagger.yaml /swagger.json /api-docs /llms.txt /llms-full.txt /capability.md *.html* /fullReport.json/* /api/* /tutorial/images/* /favicon.* /style.css /sitemap.xml
|
|
203
203
|
}
|
|
204
204
|
@allowedPOST {
|
|
205
205
|
method POST
|
package/index.html
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"dateModified": "2026-08-26",
|
|
19
19
|
"applicationCategory": "Front-end quality (i.e. web accessibility, usability, and standards conformity) testing and reporting service",
|
|
20
20
|
"isAccessibleForFree": true,
|
|
21
|
-
"description": "Testing of web pages for front-end quality (i.e. web accessibility, usability, and standards conformity) with an ensemble of
|
|
21
|
+
"description": "Testing of web pages for front-end quality (i.e. web accessibility, usability, and standards conformity) with an ensemble of 12 rule engines and provision of results at selectable levels of detail.",
|
|
22
22
|
"softwareHelp": {
|
|
23
23
|
"@type": "CreativeWork",
|
|
24
24
|
"url": "https://kilotest.com/tutorial.html"
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"WCAG 2.2 conformance evaluation",
|
|
29
29
|
"HTML5 conformance evaluation",
|
|
30
30
|
"CSS3 conformance evaluation",
|
|
31
|
-
"Ensemble of
|
|
31
|
+
"Ensemble of 12 rule engines performing more than a thousand tests",
|
|
32
32
|
"Multi-page, single-page, single-issue, and single-element reporting",
|
|
33
33
|
"Priority-ranked issue lists",
|
|
34
34
|
"Public API, connector, and tools for language-model integration"
|
|
@@ -67,10 +67,11 @@
|
|
|
67
67
|
<p>Choosing among the <a href="https://www.w3.org/WAI/test-evaluate/tools/list/">many automated rule engines</a> is hard, because they <a href="https://arxiv.org/pdf/2304.07591">vary in what and how they test</a>.</p>
|
|
68
68
|
<p><strong>Ensemble testing</strong>, instead of choosing only one, combines <strong>multiple rule engines</strong>. It can discover more issues than single-rule-engine testing, and the rule engines can check one another’s quality.</p>
|
|
69
69
|
<h2>The rule engines</h2>
|
|
70
|
-
<p>The Kilotest ensemble is <a href="https://github.com/
|
|
70
|
+
<p>The Kilotest ensemble is <a href="https://github.com/YRA-Tech/testaro?tab=readme-ov-file#dependencies">12 rule engines</a> developed by teams in Australia, Canada, Denmark, Luxembourg, Portugal, Spain, Switzerland, the United Kingdom, and the United States.</p>
|
|
71
71
|
<p>The rule engines perform about 1,300 tests in total. <a href="https://medium.com/cvs-health-tech-blog/how-to-run-a-thousand-accessibility-tests-63692ad120c3">Open-source software</a> developed initially at CVS Health drives the rule engines, tries to weed out invalid results, and distills the findings into about 380 <q>issues</q>.</p>
|
|
72
72
|
<h2>Features</h2>
|
|
73
73
|
<p>With Kilotest, you can get results of already performed tests and propose web pages for testing. (Immediate testing on request may become available in a future version.)</p>
|
|
74
|
+
<p>There is a <a href="/capability.md">capability document</a> that summarizes what Kilotest can do for human and AI-agent users.</p>
|
|
74
75
|
<h2>How to use Kilotest</h2>
|
|
75
76
|
<h3>If you are a human user on the web</h3>
|
|
76
77
|
<p>You can use Kilotest starting on this page.</p>
|
package/index.js
CHANGED
|
@@ -66,36 +66,37 @@ const answer = {
|
|
|
66
66
|
// Paths that the application is authorized to handle, by method, as glob-style patterns where * matches any sequence of characters.
|
|
67
67
|
const routes = exports.routes = {
|
|
68
68
|
GET: [
|
|
69
|
-
'
|
|
69
|
+
'*.html*',
|
|
70
70
|
'/',
|
|
71
|
-
'/index.html',
|
|
72
|
-
'/robots.txt',
|
|
73
|
-
'/openapi.yaml',
|
|
74
|
-
'/openapi.json',
|
|
75
|
-
'/swagger.yaml',
|
|
76
|
-
'/swagger.json',
|
|
77
71
|
'/api-docs',
|
|
78
|
-
'/
|
|
72
|
+
'/api/*',
|
|
73
|
+
'/capability.md',
|
|
74
|
+
'/favicon.*',
|
|
75
|
+
'/fullReport.json/*',
|
|
76
|
+
'/index.html',
|
|
79
77
|
'/llms-full.txt',
|
|
78
|
+
'/llms.txt',
|
|
79
|
+
'/mcp',
|
|
80
|
+
'/openapi.json',
|
|
81
|
+
'/openapi.yaml',
|
|
82
|
+
'/robots.txt',
|
|
80
83
|
'/sitemap.xml',
|
|
81
84
|
'/style.css',
|
|
82
|
-
'/
|
|
83
|
-
'/
|
|
84
|
-
'/tutorial/images/*'
|
|
85
|
-
'/favicon.*',
|
|
86
|
-
'*.html*'
|
|
85
|
+
'/swagger.json',
|
|
86
|
+
'/swagger.yaml',
|
|
87
|
+
'/tutorial/images/*'
|
|
87
88
|
],
|
|
88
89
|
POST: [
|
|
90
|
+
'/api/*',
|
|
89
91
|
'/mcp',
|
|
90
|
-
'/requestTest.html',
|
|
91
|
-
'/requestRetest.html/*',
|
|
92
|
-
'/recAction.html',
|
|
93
92
|
'/reannotate.html',
|
|
93
|
+
'/recAction.html',
|
|
94
94
|
'/renewWCAG.html',
|
|
95
|
+
'/requestRetest.html/*',
|
|
96
|
+
'/requestTest.html',
|
|
97
|
+
'/tutorialComment.html',
|
|
95
98
|
'/worker/job',
|
|
96
|
-
'/worker/report'
|
|
97
|
-
'/api/*',
|
|
98
|
-
'/tutorialComment.html'
|
|
99
|
+
'/worker/report'
|
|
99
100
|
]
|
|
100
101
|
};
|
|
101
102
|
const protocol = process.env.PROTOCOL || 'http';
|
|
@@ -412,6 +413,13 @@ const requestHandler = async (request, response) => {
|
|
|
412
413
|
setHeaders('text/plain', '/llms-full.txt', 'medium');
|
|
413
414
|
response.end(llmsfull);
|
|
414
415
|
}
|
|
416
|
+
// Otherwise, if it is for the summary of capabilities:
|
|
417
|
+
else if (pageName === 'capability.md') {
|
|
418
|
+
const capabilityDoc = await fs.readFile('capability.md', 'utf8');
|
|
419
|
+
// Serve it.
|
|
420
|
+
setHeaders('text/markdown', '/capability.md', 'medium');
|
|
421
|
+
response.end(capabilityDoc);
|
|
422
|
+
}
|
|
415
423
|
// Otherwise, if it is for the XML sitemap:
|
|
416
424
|
else if (pageName === 'sitemap.xml') {
|
|
417
425
|
const sitemap = await fs.readFile('sitemap.xml', 'utf8');
|
package/index.test.js
CHANGED
|
@@ -18,7 +18,7 @@ process.env.TESTARO_WORKERS = JSON.stringify({
|
|
|
18
18
|
|
|
19
19
|
// IMPORTS
|
|
20
20
|
|
|
21
|
-
const {test, before, after} = require('node:test');
|
|
21
|
+
const {test, before, beforeEach, after} = require('node:test');
|
|
22
22
|
const assert = require('node:assert/strict');
|
|
23
23
|
const http = require('node:http');
|
|
24
24
|
const fs = require('node:fs/promises');
|
|
@@ -35,7 +35,12 @@ const recsPath = path.join(fixtureDBDir, 'jobs', 'recs.json');
|
|
|
35
35
|
let server;
|
|
36
36
|
|
|
37
37
|
before(async () => {
|
|
38
|
-
|
|
38
|
+
server = http.createServer(requestHandler);
|
|
39
|
+
await new Promise(resolve => server.listen(port, () => resolve()));
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
// Restore recs.json and clean job directories before each test, so tests do not depend on execution order.
|
|
43
|
+
beforeEach(async () => {
|
|
39
44
|
await fs.writeFile(recsPath, '{}\n');
|
|
40
45
|
for (const sub of ['claimed', 'queue', 'failed']) {
|
|
41
46
|
const dir = path.join(fixtureDBDir, 'jobs', sub);
|
|
@@ -45,8 +50,6 @@ before(async () => {
|
|
|
45
50
|
await fs.unlink(path.join(dir, file)).catch(() => {});
|
|
46
51
|
}
|
|
47
52
|
}
|
|
48
|
-
server = http.createServer(requestHandler);
|
|
49
|
-
await new Promise(resolve => server.listen(port, () => resolve()));
|
|
50
53
|
});
|
|
51
54
|
|
|
52
55
|
after(async () => {
|
|
@@ -211,6 +214,13 @@ test('GET /llms-full.txt serves the LLM detailed guide as text', async () => {
|
|
|
211
214
|
assert.ok(res.body.length > 0);
|
|
212
215
|
});
|
|
213
216
|
|
|
217
|
+
test('GET /capability.md serves the capability manifest as markdown', async () => {
|
|
218
|
+
const res = await request('GET', '/capability.md');
|
|
219
|
+
assert.equal(res.statusCode, 200);
|
|
220
|
+
assert.ok(res.headers['content-type'].includes('text/markdown'));
|
|
221
|
+
assert.ok(res.body.length > 0);
|
|
222
|
+
});
|
|
223
|
+
|
|
214
224
|
test('GET /sitemap.xml serves the sitemap as XML', async () => {
|
|
215
225
|
const res = await request('GET', '/sitemap.xml');
|
|
216
226
|
assert.equal(res.statusCode, 200);
|
|
@@ -251,7 +261,7 @@ test('GET /api/listViolators/linkNoText/260101T0000/mix returns JSON', async ()
|
|
|
251
261
|
assert.equal(body['tool name'], 'listViolators');
|
|
252
262
|
});
|
|
253
263
|
|
|
254
|
-
test('GET /api/listDiagnoses/0/linkNoText/260101T0000/mix returns JSON', async () => {
|
|
264
|
+
test('GET /api/listDiagnoses/0/linkNoText/260101T0000/mix returns JSON', {timeout: 500}, async () => {
|
|
255
265
|
const res = await request('GET', '/api/listDiagnoses/0/linkNoText/260101T0000/mix');
|
|
256
266
|
assert.equal(res.statusCode, 200);
|
|
257
267
|
const body = jsonBody(res);
|
|
@@ -265,7 +275,7 @@ test('GET /api/getReport/260101T0000/mix returns JSON with the report', async ()
|
|
|
265
275
|
assert.equal(body['tool name'], 'getReport');
|
|
266
276
|
});
|
|
267
277
|
|
|
268
|
-
test('GET /api/invalidService returns an error', async () => {
|
|
278
|
+
test('GET /api/invalidService returns an error', {timeout: 500}, async () => {
|
|
269
279
|
const res = await request('GET', '/api/invalidService');
|
|
270
280
|
assert.equal(res.statusCode, 400);
|
|
271
281
|
const body = jsonBody(res);
|
|
@@ -281,7 +291,7 @@ test('GET /fullReport.json/260101T0000/mix downloads the report as JSON', async
|
|
|
281
291
|
assert.ok(body.target);
|
|
282
292
|
});
|
|
283
293
|
|
|
284
|
-
test('GET /fullReport.json/invalid/invalid returns an error page', async () => {
|
|
294
|
+
test('GET /fullReport.json/invalid/invalid returns an error page', {timeout: 500}, async () => {
|
|
285
295
|
const res = await request('GET', '/fullReport.json/invalid/invalid');
|
|
286
296
|
assert.equal(res.statusCode, 400);
|
|
287
297
|
assert.ok(res.headers['content-type'].includes('text/html'));
|
|
@@ -382,7 +392,7 @@ test('POST /requestRetest.html/invalid/invalid with invalid data returns an erro
|
|
|
382
392
|
assert.ok(res.body.includes('Invalid retest recommendation'));
|
|
383
393
|
});
|
|
384
394
|
|
|
385
|
-
test('POST /api/requestTest with valid JSON returns a JSON response', async () => {
|
|
395
|
+
test('POST /api/requestTest with valid JSON returns a JSON response', {timeout: 500}, async () => {
|
|
386
396
|
const res = await request('POST', '/api/requestTest', {
|
|
387
397
|
description: `API Test Page ${uniqueStamp}`,
|
|
388
398
|
URL: `https://example.com/api-test-${uniqueStamp}`,
|
|
@@ -502,7 +512,7 @@ test('serveError sends HTML for human requests (isHumanUser = true)', async () =
|
|
|
502
512
|
|
|
503
513
|
// TESTS: recAction.html
|
|
504
514
|
|
|
505
|
-
test('POST /recAction.html with invalid auth code returns an error', async () => {
|
|
515
|
+
test('POST /recAction.html with invalid auth code returns an error', {timeout: 500}, async () => {
|
|
506
516
|
const res = await formRequest('POST', '/recAction.html', {
|
|
507
517
|
target: 'https://example.com\tTest Page',
|
|
508
518
|
authCode: 'wrong-code'
|
|
@@ -542,7 +552,7 @@ test('GET /fullReport.json/260101T0007/hid returns an abuse error for a hidden r
|
|
|
542
552
|
assert.ok(res.body.includes('Invalid request'));
|
|
543
553
|
});
|
|
544
554
|
|
|
545
|
-
test('POST /recAction.html with valid auth code and approval returns HTML', async () => {
|
|
555
|
+
test('POST /recAction.html with valid auth code and approval returns HTML', {timeout: 500}, async () => {
|
|
546
556
|
await fs.writeFile(recsPath, '{}\n');
|
|
547
557
|
await formRequest('POST', '/requestTest.html', {
|
|
548
558
|
what: `Approval Test Page ${uniqueStamp}`,
|
|
@@ -619,7 +629,7 @@ test('POST /renewWCAG.html with valid auth code serves the answer page', async (
|
|
|
619
629
|
}
|
|
620
630
|
});
|
|
621
631
|
|
|
622
|
-
test('POST /tutorialComment.html with empty content returns a JSON error', async () => {
|
|
632
|
+
test('POST /tutorialComment.html with empty content returns a JSON error', {timeout: 500}, async () => {
|
|
623
633
|
const res = await request('POST', '/tutorialComment.html', {
|
|
624
634
|
content: ''
|
|
625
635
|
});
|
|
@@ -691,7 +701,7 @@ test('POST /worker/job with no colon in decoded credentials returns 401', async
|
|
|
691
701
|
|
|
692
702
|
// TESTS: processJobRequest branches
|
|
693
703
|
|
|
694
|
-
test('POST /worker/job with a claimed job assigned to the worker returns an error and reclassifies the job', async () => {
|
|
704
|
+
test('POST /worker/job with a claimed job assigned to the worker returns an error and reclassifies the job', {timeout: 500}, async () => {
|
|
695
705
|
// Create a claimed job assigned to Worker One.
|
|
696
706
|
const claimedDir = path.join(fixtureDBDir, 'jobs', 'claimed');
|
|
697
707
|
const failedDir = path.join(fixtureDBDir, 'jobs', 'failed');
|
|
@@ -712,6 +722,8 @@ test('POST /worker/job with a claimed job assigned to the worker returns an erro
|
|
|
712
722
|
// The worker should get an error about the incomplete job.
|
|
713
723
|
const body = jsonBody(res);
|
|
714
724
|
assert.ok(body.error.message.includes('has not completed job'));
|
|
725
|
+
// Wait for the async rename to complete.
|
|
726
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
715
727
|
// The job should have been moved to failed.
|
|
716
728
|
const failedExists = await fs.access(failedJobPath).then(() => true).catch(() => false);
|
|
717
729
|
assert.ok(failedExists, 'Job should be moved to failed directory');
|
|
@@ -758,7 +770,7 @@ test('POST /worker/job with a queued job assigns it to the worker', {timeout: 50
|
|
|
758
770
|
|
|
759
771
|
// TESTS: worker/report valid submission
|
|
760
772
|
|
|
761
|
-
test('POST /worker/report with valid authentication and valid claimed job processes the report', async () => {
|
|
773
|
+
test('POST /worker/report with valid authentication and valid claimed job processes the report', {timeout: 500}, async () => {
|
|
762
774
|
// Use a unique job ID that does not conflict with existing fixtures.
|
|
763
775
|
const jobID = '990101T0000-tst';
|
|
764
776
|
const reportPath = path.join(fixtureDBDir, 'reports', `${jobID}.json`);
|
|
@@ -851,6 +863,18 @@ for (const pagePath of htmlPagePaths) {
|
|
|
851
863
|
});
|
|
852
864
|
}
|
|
853
865
|
|
|
866
|
+
test('GET /enqueueForm.html shows recommendations when recs.json has entries', async () => {
|
|
867
|
+
await fs.writeFile(recsPath, JSON.stringify({
|
|
868
|
+
'https://example.com/enqueue-test': [
|
|
869
|
+
{what: 'Enqueue Test Page', why: 'Needs testing for accessibility'}
|
|
870
|
+
]
|
|
871
|
+
}));
|
|
872
|
+
const res = await request('GET', '/enqueueForm.html');
|
|
873
|
+
assert.equal(res.statusCode, 200);
|
|
874
|
+
assert.ok(res.body.includes('https://example.com/enqueue-test'));
|
|
875
|
+
assert.ok(res.body.includes('Enqueue Test Page'));
|
|
876
|
+
});
|
|
877
|
+
|
|
854
878
|
test('GET /listRules.html returns an error page when called without arguments', async () => {
|
|
855
879
|
const res = await request('GET', '/listRules.html');
|
|
856
880
|
assert.equal(res.statusCode, 400);
|
|
@@ -891,7 +915,7 @@ test('POST /mcp returns a response from the MCP handler', async () => {
|
|
|
891
915
|
|
|
892
916
|
// TESTS: answer error branches
|
|
893
917
|
|
|
894
|
-
test('POST /requestTest.html with valid format but duplicate URL returns an answer error', async () => {
|
|
918
|
+
test('POST /requestTest.html with valid format but duplicate URL returns an answer error', {timeout: 500}, async () => {
|
|
895
919
|
await fs.writeFile(recsPath, '{}\n');
|
|
896
920
|
// First request to create the recommendation.
|
|
897
921
|
await formRequest('POST', '/requestTest.html', {
|
|
@@ -909,7 +933,7 @@ test('POST /requestTest.html with valid format but duplicate URL returns an answ
|
|
|
909
933
|
assert.ok(res.body.includes('Duplicate recommendation'));
|
|
910
934
|
});
|
|
911
935
|
|
|
912
|
-
test('POST /requestRetest.html with valid format but duplicate retest returns an answer error', async () => {
|
|
936
|
+
test('POST /requestRetest.html with valid format but duplicate retest returns an answer error', {timeout: 500}, async () => {
|
|
913
937
|
await fs.writeFile(recsPath, '{}\n');
|
|
914
938
|
// First retest to create the recommendation.
|
|
915
939
|
await formRequest('POST', '/requestRetest.html/260202T0000/new', {
|
|
@@ -923,7 +947,7 @@ test('POST /requestRetest.html with valid format but duplicate retest returns an
|
|
|
923
947
|
assert.ok(res.body.includes('Duplicate recommendation'));
|
|
924
948
|
});
|
|
925
949
|
|
|
926
|
-
test('POST /recAction.html with valid auth code and approval of a duplicate returns an error', async () => {
|
|
950
|
+
test('POST /recAction.html with valid auth code and approval of a duplicate returns an error', {timeout: 500}, async () => {
|
|
927
951
|
await fs.writeFile(recsPath, '{}\n');
|
|
928
952
|
// Create a recommendation.
|
|
929
953
|
await formRequest('POST', '/requestTest.html', {
|
|
@@ -1054,7 +1078,7 @@ test('startServer starts an HTTP server when protocol is http', async () => {
|
|
|
1054
1078
|
}
|
|
1055
1079
|
});
|
|
1056
1080
|
|
|
1057
|
-
test('startServer starts an HTTPS server when protocol is https', {timeout:
|
|
1081
|
+
test('startServer starts an HTTPS server when protocol is https', {timeout: 2000}, async () => {
|
|
1058
1082
|
// Generate a self-signed certificate for the test using openssl.
|
|
1059
1083
|
const {execSync} = require('node:child_process');
|
|
1060
1084
|
const os = require('node:os');
|