@jrpool/kilotest 52.0.0 → 57.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/.devin/config.local.json +4 -1
  2. package/.github/workflows/ci.yml +3 -0
  3. package/README.md +1 -1
  4. package/api/awaitTest.test.ts +114 -0
  5. package/api/awaitTest.ts +108 -0
  6. package/api/listReports.ts +2 -2
  7. package/api/orderNewTest.test.ts +213 -0
  8. package/api/orderNewTest.ts +145 -0
  9. package/api/orderRetest.test.ts +179 -0
  10. package/api/orderRetest.ts +119 -0
  11. package/api/requestFeature.test.ts +36 -0
  12. package/api/requestFeature.ts +26 -4
  13. package/api/{requestTest.test.ts → requestNewTest.test.ts} +57 -23
  14. package/api/{requestTest.ts → requestNewTest.ts} +31 -36
  15. package/api/requestRetest.test.ts +24 -6
  16. package/api/requestRetest.ts +10 -29
  17. package/api/routes.test.ts +7 -4
  18. package/api/routes.ts +45 -7
  19. package/api/schemas.ts +120 -10
  20. package/api/util.ts +59 -2
  21. package/api/version.ts +6 -2
  22. package/capability.md +9 -6
  23. package/docs/AI-TOOLS.md +3 -3
  24. package/docs/SERVICE.md +19 -8
  25. package/env.example +39 -0
  26. package/index.test.ts +146 -24
  27. package/index.ts +129 -36
  28. package/lhm.plugin.json +1 -1
  29. package/linkValidation.test.ts +2 -2
  30. package/llms-full.txt +35 -4
  31. package/llms.txt +6 -3
  32. package/mcp.test.ts +109 -12
  33. package/mcp.ts +146 -26
  34. package/memory/MEMORY.md +1 -0
  35. package/memory/plan_issue_111_instant_test.md +198 -0
  36. package/openapi.yaml +456 -9
  37. package/package.json +2 -1
  38. package/routes.test.ts +1 -1
  39. package/routing.test.ts +2 -2
  40. package/scripts/generate-openapi.ts +1 -1
  41. package/server.json +1 -1
  42. package/sitemap.xml +1 -1
  43. package/smokeTest.ts +2 -2
  44. package/util.test.ts +486 -2
  45. package/util.ts +276 -45
  46. package/web/listReports/index.html +1 -1
  47. package/web/listReports/listReports.test.ts +1 -1
  48. package/web/{requestTest → requestNewTest}/index.ts +10 -1
  49. package/web/{requestTest/requestTest.test.ts → requestNewTest/requestNewTest.test.ts} +18 -2
  50. package/web/{requestTestForm → requestNewTestForm}/index.html +1 -1
  51. package/web/{requestTestForm/requestTestForm.test.ts → requestNewTestForm/requestNewTestForm.test.ts} +6 -6
  52. package/web/requestRetest/index.ts +9 -0
  53. package/web/requestRetest/requestRetest.test.ts +14 -0
  54. package/web/tutorialAI/index.ts +25 -3
  55. package/web/tutorialAI/tutorialAI.test.ts +32 -0
  56. package/web/tutorialWeb/index.ts +25 -3
  57. package/web/tutorialWeb/tutorialWeb.test.ts +32 -0
  58. /package/web/{requestTest → requestNewTest}/index.html +0 -0
  59. /package/web/{requestTestForm → requestNewTestForm}/index.ts +0 -0
package/index.ts CHANGED
@@ -17,6 +17,7 @@ import {
17
17
  getObject,
18
18
  getPOSTData,
19
19
  getReport,
20
+ isAllowedRedirectTarget,
20
21
  isAllowedReport,
21
22
  isAllowedTarget,
22
23
  isReportError,
@@ -66,8 +67,8 @@ import {answer as renewWCAG} from './web/renewWCAG/index.ts';
66
67
  import {answer as renewWCAGForm} from './web/renewWCAGForm/index.ts';
67
68
  import {answer as requestRetestPage} from './web/requestRetest/index.ts';
68
69
  import {answer as requestRetestForm} from './web/requestRetestForm/index.ts';
69
- import {answer as requestTestPage} from './web/requestTest/index.ts';
70
- import {answer as requestTestForm} from './web/requestTestForm/index.ts';
70
+ import {answer as requestNewTestPage} from './web/requestNewTest/index.ts';
71
+ import {answer as requestNewTestForm} from './web/requestNewTestForm/index.ts';
71
72
  import {answer as rewindReportsForm} from './web/rewindReportsForm/index.ts';
72
73
  import {answer as showHiddenReportsForm} from './web/showHiddenReportsForm/index.ts';
73
74
  import {answer as unhideReportForm} from './web/unhideReportForm/index.ts';
@@ -78,7 +79,10 @@ import {response as listReportsAPI} from './api/listReports.ts';
78
79
  import {response as listViolatorsAPI} from './api/listViolators.ts';
79
80
  import {response as requestFeatureAPI} from './api/requestFeature.ts';
80
81
  import {response as requestRetestAPI} from './api/requestRetest.ts';
81
- import {response as requestTestAPI} from './api/requestTest.ts';
82
+ import {response as requestNewTestAPI} from './api/requestNewTest.ts';
83
+ import {response as orderNewTestAPI} from './api/orderNewTest.ts';
84
+ import {response as orderRetestAPI} from './api/orderRetest.ts';
85
+ import {response as awaitTestAPI} from './api/awaitTest.ts';
82
86
 
83
87
  // ENVIRONMENT
84
88
 
@@ -123,8 +127,8 @@ const answer: {
123
127
  renewWCAGForm: PageHandler;
124
128
  requestRetest: PageHandler;
125
129
  requestRetestForm: PageHandler;
126
- requestTest: PageHandler;
127
- requestTestForm: PageHandler;
130
+ requestNewTest: PageHandler;
131
+ requestNewTestForm: PageHandler;
128
132
  rewindReportsForm: PageHandler;
129
133
  showHiddenReportsForm: PageHandler;
130
134
  unhideReportForm: PageHandler;
@@ -153,8 +157,8 @@ const answer: {
153
157
  renewWCAGForm,
154
158
  requestRetest: requestRetestPage,
155
159
  requestRetestForm,
156
- requestTest: requestTestPage,
157
- requestTestForm,
160
+ requestNewTest: requestNewTestPage,
161
+ requestNewTestForm,
158
162
  rewindReportsForm,
159
163
  showHiddenReportsForm,
160
164
  unhideReportForm
@@ -169,7 +173,10 @@ const apiRespond: {
169
173
  listViolators: ApiResponder;
170
174
  requestFeature: ApiResponder;
171
175
  requestRetest: ApiResponder;
172
- requestTest: ApiResponder;
176
+ requestNewTest: ApiResponder;
177
+ orderNewTest: ApiResponder;
178
+ orderRetest: ApiResponder;
179
+ awaitTest: ApiResponder;
173
180
  } = {
174
181
  getReport: getReportAPI,
175
182
  listDiagnoses: listDiagnosesAPI,
@@ -178,7 +185,10 @@ const apiRespond: {
178
185
  listViolators: listViolatorsAPI,
179
186
  requestFeature: requestFeatureAPI,
180
187
  requestRetest: requestRetestAPI,
181
- requestTest: requestTestAPI
188
+ requestNewTest: requestNewTestAPI,
189
+ orderNewTest: orderNewTestAPI,
190
+ orderRetest: orderRetestAPI,
191
+ awaitTest: awaitTestAPI
182
192
  };
183
193
 
184
194
  // CONSTANTS
@@ -225,7 +235,7 @@ export const routes = {
225
235
  '/requestAction.html',
226
236
  '/renewWCAG.html',
227
237
  '/requestRetest.html/*',
228
- '/requestTest.html',
238
+ '/requestNewTest.html',
229
239
  '/tutorialWebComment.html',
230
240
  '/unhideReportForm.html',
231
241
  '/worker/job',
@@ -275,12 +285,18 @@ const noDirectGetPages = new Set(['unhideReportForm']);
275
285
  // engines and other crawlers are still requesting. Redirecting them lets crawlers update
276
286
  // their own indexes, rather than leaving them to repeat the same dead request indefinitely.
277
287
  // TEMPORARY: retire this redirect after 2027-04-01 once crawlers have re-indexed.
288
+ // requestTest and requestTestForm were renamed to requestNewTest and requestNewTestForm
289
+ // on 2026-09-28 (see the newTest/retest naming-ambiguity fix), to distinguish the
290
+ // new-test-only page from bare "test" language used elsewhere.
291
+ // TEMPORARY: retire this redirect after 2027-03-28 once crawlers have re-indexed.
278
292
  const renamedPagePrefixes: Record<string, string> = {
279
293
  diagnoses: 'listDiagnoses',
280
294
  reportIssue: 'listViolators',
281
295
  reportIssues: 'listIssues',
282
296
  rules: 'listRules',
283
- targets: 'listReports'
297
+ targets: 'listReports',
298
+ requestTest: 'requestNewTest',
299
+ requestTestForm: 'requestNewTestForm'
284
300
  };
285
301
  const jobLock = createLock();
286
302
 
@@ -484,26 +500,54 @@ const processJobRequest = async (request: IncomingMessage, response: ServerRespo
484
500
  if (queuedJobNames.length) {
485
501
  const oldestJobName = queuedJobNames[0]!;
486
502
  // Get the first one.
487
- const firstJob = await getObject(path.join(queuePath(), oldestJobName)) as {id: string, sources: {worker: string}, target: {what: string}};
488
- // Add the public worker name to the job, in a property Testaro does not read or alter.
489
- firstJob.sources.worker = workerName;
490
- console.log(
491
- `Job ${firstJob.id} (${firstJob.target.what}) is being sent to the worker.`
492
- );
493
- // Assign the job to the worker.
494
- response.writeHead(200, {
495
- 'content-type': 'application/json; charset=utf-8'
496
- });
497
- response.end(JSON.stringify(firstJob));
498
- const messageEnd
499
- = `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the worker`;
500
- console.log(`${messageStart}${messageEnd}`);
501
- // Save the job in the claimed-jobs directory.
502
- await fs.writeFile(
503
- path.join(claimedPath(), oldestJobName), getJSON(firstJob)
504
- );
505
- // Delete it from the queue.
506
- await fs.unlink(path.join(queuePath(), oldestJobName));
503
+ const firstJob = await getObject(path.join(queuePath(), oldestJobName)) as {id: string, sources: {worker: string}, target: {what: string, url: string}};
504
+ // If the job's target is no longer allowed, whether because its host now resolves
505
+ // to a disallowed address or, more likely, because the host now redirects to one
506
+ // (both possible if the target host was reconfigured, e.g. by an attacker, after
507
+ // the request was accepted; a fetch-based check is used, rather than the cheaper
508
+ // DNS-only isAllowedTarget, because this runs only when a job is actually queued,
509
+ // which is rare, and a redirect introduced after acceptance is the likelier attack):
510
+ if (!(await isAllowedRedirectTarget(firstJob.target.url))) {
511
+ console.error(`ERROR: Queued job ${firstJob.id} is on a target this deployment does not allow`);
512
+ // Alert a manager, since this indicates an attempted or accidental SSRF
513
+ // rather than an ordinary usability problem with the target.
514
+ await sendAlert(
515
+ 'Kilotest: queued job on disallowed target rejected',
516
+ `Job ${firstJob.id} (${firstJob.target.what}) was never assigned to a worker because its target no longer resolves, directly or via a redirect, to an address this deployment allows (e.g. a private, loopback, or link-local address). The job was reclassified as failed.`
517
+ );
518
+ // Reclassify the job as failed, instead of assigning it or leaving it queued.
519
+ await fs.rename(
520
+ path.join(queuePath(), oldestJobName), path.join(failedPath(), oldestJobName)
521
+ );
522
+ // Respond as if no job were queued, rather than revealing anything about the rejected job.
523
+ response.writeHead(200, {
524
+ 'content-type': 'application/json; charset=utf-8'
525
+ });
526
+ response.end(JSON.stringify({}));
527
+ console.log(`${messageStart}but the queued job was on a disallowed target`);
528
+ }
529
+ // Otherwise, i.e. if the job's target is still allowed:
530
+ else {
531
+ // Add the public worker name to the job, in a property Testaro does not read or alter.
532
+ firstJob.sources.worker = workerName;
533
+ console.log(
534
+ `Job ${firstJob.id} (${firstJob.target.what}) is being sent to the worker.`
535
+ );
536
+ // Assign the job to the worker.
537
+ response.writeHead(200, {
538
+ 'content-type': 'application/json; charset=utf-8'
539
+ });
540
+ response.end(JSON.stringify(firstJob));
541
+ const messageEnd
542
+ = `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the worker`;
543
+ console.log(`${messageStart}${messageEnd}`);
544
+ // Save the job in the claimed-jobs directory.
545
+ await fs.writeFile(
546
+ path.join(claimedPath(), oldestJobName), getJSON(firstJob)
547
+ );
548
+ // Delete it from the queue.
549
+ await fs.unlink(path.join(queuePath(), oldestJobName));
550
+ }
507
551
  }
508
552
  // Otherwise, i.e. if no jobs are queued:
509
553
  else {
@@ -898,7 +942,7 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
898
942
  await serveError({message: 'ERROR: Unreadable request body'}, response, true);
899
943
  }
900
944
  // If the request is a test request:
901
- else if (pageName === 'requestTest.html') {
945
+ else if (pageName === 'requestNewTest.html') {
902
946
  const {description, url, why} = postData as {description?: string; url: string; why?: string};
903
947
  // If the request is valid:
904
948
  if (description && isURL(url) && why && await isAllowedTarget(url)) {
@@ -912,7 +956,7 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
912
956
  // Serve headers for a response.
913
957
  setHeaders('text/html', pathname, 'ultra');
914
958
  // Get the answer data.
915
- const answerData = await answer.requestTest(description, url, why);
959
+ const answerData = await answer.requestNewTest(description, url, why);
916
960
  // If they are valid:
917
961
  if (answerData.status === 'ok') {
918
962
  // Serve the answer page.
@@ -1197,15 +1241,26 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
1197
1241
  // Get the segments of the path after api.
1198
1242
  const segments = pathTail.split('/');
1199
1243
  // If the service is to receive a test request:
1200
- if (segments[0] === 'requestTest') {
1244
+ if (segments[0] === 'requestNewTest') {
1201
1245
  const {description, URL, reason} = postData as {description: string; URL: string; reason: string};
1202
1246
  // Get the response body.
1203
- const responseBody = await apiRespond.requestTest([description, URL, reason]);
1204
- await recordPageMetric('apiOperations', 'requestTest');
1247
+ const responseBody = await apiRespond.requestNewTest([description, URL, reason]);
1248
+ await recordPageMetric('apiOperations', 'requestNewTest');
1205
1249
  // Send it.
1206
1250
  setHeaders('application/json', null, 'ultra');
1207
1251
  response.end(JSON.stringify(responseBody));
1208
1252
  }
1253
+ // Otherwise, if the request uses the old, pre-rename requestTest path name: redirect
1254
+ // permanently to the new path name. requestTest takes no path parameters (its
1255
+ // arguments are body fields), so there is no further path to preserve, unlike the
1256
+ // renamedPagePrefixes GET-page redirects above. 308 (not 301) is used here because
1257
+ // 301 is defined to let clients rewrite a POST redirect as a GET, which would drop
1258
+ // this request's body; 308 preserves both the method and the body.
1259
+ // TEMPORARY: retire this redirect after 2027-03-28 once callers have updated.
1260
+ else if (segments[0] === 'requestTest') {
1261
+ response.writeHead(308, {Location: '/api/requestNewTest'});
1262
+ response.end();
1263
+ }
1209
1264
  // Otherwise, if the service is to receive a retest request:
1210
1265
  else if (segments[0] === 'requestRetest') {
1211
1266
  const {reason} = postData as {reason: string};
@@ -1226,6 +1281,44 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
1226
1281
  setHeaders('application/json', null, 'ultra');
1227
1282
  response.end(JSON.stringify(responseBody));
1228
1283
  }
1284
+ // Otherwise, if the service is to order an immediate, automatically approved test:
1285
+ else if (segments[0] === 'orderNewTest') {
1286
+ const {description, URL, reason} = postData as {description: string; URL: string; reason: string};
1287
+ // Get the response body.
1288
+ const responseBody = await apiRespond.orderNewTest([description, URL, reason]);
1289
+ await recordPageMetric('apiOperations', 'orderNewTest');
1290
+ // Send it.
1291
+ setHeaders('application/json', null, 'ultra');
1292
+ response.end(JSON.stringify(responseBody));
1293
+ }
1294
+ // Otherwise, if the request uses the old, pre-rename orderTest path name: redirect
1295
+ // permanently to the new path name. orderTest likewise takes no path parameters.
1296
+ // See the requestTest redirect above for why 308 (not 301) is used for these
1297
+ // POST-only API redirects.
1298
+ // TEMPORARY: retire this redirect after 2027-03-28 once callers have updated.
1299
+ else if (segments[0] === 'orderTest') {
1300
+ response.writeHead(308, {Location: '/api/orderNewTest'});
1301
+ response.end();
1302
+ }
1303
+ // Otherwise, if the service is to order an immediate, automatically approved retest:
1304
+ else if (segments[0] === 'orderRetest') {
1305
+ const {reason} = postData as {reason: string};
1306
+ // Get the response body.
1307
+ const responseBody = await apiRespond.orderRetest(segments.slice(1).concat(reason));
1308
+ await recordPageMetric('apiOperations', 'orderRetest');
1309
+ // Send it.
1310
+ setHeaders('application/json', null, 'ultra');
1311
+ response.end(JSON.stringify(responseBody));
1312
+ }
1313
+ // Otherwise, if the service is to wait for an ordered job to complete:
1314
+ else if (segments[0] === 'awaitTest') {
1315
+ // Get the response body.
1316
+ const responseBody = await apiRespond.awaitTest(segments.slice(1));
1317
+ await recordPageMetric('apiOperations', 'awaitTest');
1318
+ // Send it.
1319
+ setHeaders('application/json', null, 'ultra');
1320
+ response.end(JSON.stringify(responseBody));
1321
+ }
1229
1322
  // Otherwise, i.e. if the service is invalid:
1230
1323
  else {
1231
1324
  await serveError(
package/lhm.plugin.json CHANGED
@@ -2297,7 +2297,7 @@
2297
2297
  }
2298
2298
  },
2299
2299
  {
2300
- "name": "requestTest",
2300
+ "name": "requestNewTest",
2301
2301
  "description": "Process my request to test a page about which no report is available yet.",
2302
2302
  "inputSchema": {
2303
2303
  "type": "object",
@@ -48,8 +48,8 @@ const htmlRoutes = [
48
48
  '/listViolators.html',
49
49
  '/listRules.html',
50
50
  '/manage.html',
51
- '/requestTest.html',
52
- '/requestTestForm.html',
51
+ '/requestNewTest.html',
52
+ '/requestNewTestForm.html',
53
53
  '/requestRetest.html',
54
54
  '/requestRetestForm.html',
55
55
  '/reannotate.html',
package/llms-full.txt CHANGED
@@ -6,7 +6,7 @@ Kilotest is an application that performs ensemble testing of web pages for front
6
6
 
7
7
  ## What Kilotest does for language models
8
8
 
9
- Kilotest is deployed on the public Internet as a service at `https://kilotest.com`. Kilotest can test only web pages that can be accessed from the public Internet and that are not protected by authentication or other access controls. Kilotest has not implemented any mechanism for testing private, internal, password-protected, or otherwise restricted pages.
9
+ Kilotest is deployed on the public Internet as a service at `https://kilotest.com`. That deployment of Kilotest can test only web pages that can be accessed by its Testaro worker with a URL. Generally those are pages on the public Internet that are not protected by authentication or other access controls.
10
10
 
11
11
  Without calling any tool, an LLM can, under some conditions, load web pages and use its knowledge to assess successes and failures of front-end quality. But such an assessment is almost always:
12
12
 
@@ -63,7 +63,7 @@ Kilotest offers a comprehensive suite of capabilities to users via its web UI:
63
63
  - Get statistics about issues reported in one report: `https://kilotest.com/listIssues.html/{timeStamp}/{jobID}`
64
64
  - Get details about one issue reported in one report: `https://kilotest.com/listViolators.html/{issueID}/{timeStamp}/{jobID}`
65
65
  - Get diagnoses by rule engines of rule violations for one HTML element exhibiting one issue in one report: `https://kilotest.com/listDiagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
66
- - [Recommend the testing of a not-yet-tested page](https://kilotest.com/requestTestForm.html)
66
+ - [Recommend the testing of a not-yet-tested page](https://kilotest.com/requestNewTestForm.html)
67
67
  - Recommend the retesting of a previously tested page: `https://kilotest.com/requestRetestForm.html/{timeStamp}/{jobID}`
68
68
  - [Get statistics about frequently reported issues across all pages](https://kilotest.com/listTopIssues.html)
69
69
  - [Get a list of the operations that Kilotest managers are authorized to request](https://kilotest.com/manage.html)
@@ -108,10 +108,10 @@ Kilotest offers these tools:
108
108
  - `timeStamp`: initial segment of report identifier
109
109
  - `jobID`: final segment of report identifier
110
110
  - source of parameters: response to a `listViolators` request
111
- - `requestTest`
111
+ - `requestNewTest`
112
112
  - method: `POST`
113
113
  - purpose: request testing of one web page about which no report is available
114
- - path: `/api/requestTest`
114
+ - path: `/api/requestNewTest`
115
115
  - payload properties
116
116
  - `description`: description of the page to be tested
117
117
  - `URL`: URL of the page to be tested
@@ -132,6 +132,35 @@ Kilotest offers these tools:
132
132
  - path: `/api/requestFeature`
133
133
  - payload properties
134
134
  - `feature`: description and motivation of the feature improvement or the new feature
135
+ - `orderNewTest`
136
+ - method: `POST`
137
+ - purpose: order immediate, automatically approved testing of one web page about which no report is available, instead of waiting for manual approval
138
+ - path: `/api/orderNewTest`
139
+ - payload properties
140
+ - `description`: description of the page to be tested
141
+ - `URL`: URL of the page to be tested
142
+ - `reason`: reason for testing the page
143
+ - response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued
144
+ - `orderRetest`
145
+ - method: `POST`
146
+ - purpose: order immediate, automatically approved retesting of one web page about which a report is available, instead of waiting for manual approval
147
+ - path: `/api/orderRetest/{timeStamp}/{jobID}`
148
+ - parameters
149
+ - `timeStamp`: initial segment of report identifier
150
+ - `jobID`: final segment of report identifier
151
+ - source of parameters: response to a `listIssues` request
152
+ - payload properties
153
+ - `reason`: reason for retesting the page
154
+ - response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued
155
+ - `awaitTest`
156
+ - method: `POST`
157
+ - purpose: wait for a job ordered via `orderNewTest` or `orderRetest` to complete; blocks until the report is ready, the job fails, or a maximum wait time elapses, then returns
158
+ - path: `/api/awaitTest/{timeStamp}/{jobID}`
159
+ - parameters
160
+ - `timeStamp`: initial segment of the report identifier returned by `orderNewTest` or `orderRetest`
161
+ - `jobID`: final segment of the report identifier returned by `orderNewTest` or `orderRetest`
162
+ - source of parameters: response to an `orderNewTest` or `orderRetest` request
163
+ - caveat: testing typically takes about 2 minutes and occasionally up to 4 minutes; sends no interim notices while waiting
135
164
 
136
165
  ## After a test, retest, or feature request
137
166
 
@@ -147,6 +176,8 @@ When you request a test or a retest:
147
176
  - check test completion by calling `listReports` and looking for the page and completion date in the list
148
177
  - you advise the user to check in approximately 1 day for the test results on the Kilotest website, starting at `https://kilotest.com/listReports.html`
149
178
 
179
+ For immediate testing without waiting for manual approval, use `orderNewTest` or `orderRetest` instead of `requestNewTest` or `requestRetest`. Each enqueues the job right away and returns a report identifier along with a question about whether to wait for completion. To wait, call `awaitTest` with that identifier; it blocks until the report is ready, the job fails, or a maximum wait time elapses. To check later instead, call `listIssues` with the same identifier.
180
+
150
181
  An [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml) is available.
151
182
 
152
183
  ## How to report results honestly to users
package/llms.txt CHANGED
@@ -1,6 +1,6 @@
1
1
  # Kilotest
2
2
 
3
- Kilotest helps language models and AI platforms to give thorough, accurate and truthful answers when users ask about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. Kilotest integrates an ensemble of twelve independent rule engines to test any public web page. Kilotest creates a structured report of the results and saves the report for future reference. Kilotest API endpoints and MCP tools can provide multipage, single-page, and element-level test results.
3
+ Kilotest helps language models and AI platforms to give thorough, accurate and truthful answers when users ask about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. Kilotest integrates an ensemble of twelve independent rule engines to test a web page. Kilotest creates a structured report of the results and saves the report for future reference. Kilotest API endpoints and MCP tools can provide multipage, single-page, and element-level test results.
4
4
 
5
5
  ## API
6
6
 
@@ -9,9 +9,12 @@ Kilotest helps language models and AI platforms to give thorough, accurate and t
9
9
  - List all elements exhibiting one issue in one report: `/api/listViolators/{issueID}/{timeStamp}/{jobID}` (`GET`)
10
10
  - List all diagnoses of one element exhibiting one issue in one report: `/api/listDiagnoses/{catalogIndex}/{issueID}/{timeStamp}/{jobID}` (`GET`)
11
11
  - Get one full report in JSON: `/api/getReport/{timeStamp}/{jobID}` (`GET`)
12
- - Request testing of a web page: `/api/requestTest` (`POST`)
12
+ - Request testing of a web page: `/api/requestNewTest` (`POST`)
13
13
  - Request retesting of a web page: `/api/requestRetest/{timeStamp}/{jobID}` (`POST`)
14
14
  - Request adding or improving a feature: `/api/requestFeature` (`POST`)
15
+ - Order immediate, automatically approved testing of a web page: `/api/orderNewTest` (`POST`)
16
+ - Order immediate, automatically approved retesting of a web page: `/api/orderRetest/{timeStamp}/{jobID}` (`POST`)
17
+ - Wait for a job ordered via orderNewTest or orderRetest to complete: `/api/awaitTest/{timeStamp}/{jobID}` (`POST`)
15
18
 
16
19
  ## MCP Server
17
20
 
@@ -24,7 +27,7 @@ Kilotest helps language models and AI platforms to give thorough, accurate and t
24
27
  - List all elements exhibiting one issue in one report: `https://kilotest.com/listViolators.html/{issueID}/{timeStamp}/{jobID}`
25
28
  - List all diagnoses of one element exhibiting one issue in one report: `https://kilotest.com/listDiagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
26
29
  - Get one full report in JSON: `https://kilotest.com/fullReport.json/{timeStamp}/{jobID}`
27
- - [Request testing of a web page](https://kilotest.com/requestTestForm.html)
30
+ - [Request testing of a web page](https://kilotest.com/requestNewTestForm.html)
28
31
  - Request retesting of a web page: `https://kilotest.com/requestRetestForm.html/{timeStamp}/{jobID}`
29
32
  - [List the operations that Kilotest managers are authorized to request](https://kilotest.com/manage.html)
30
33
  - [Provide statistics about frequently reported issues across all pages](https://kilotest.com/listTopIssues.html)
package/mcp.test.ts CHANGED
@@ -11,6 +11,8 @@ import http from 'node:http';
11
11
  import {mcpPath, createMCPServer, handleMCP} from './mcp.ts';
12
12
  import {metricsPath} from './util.ts';
13
13
  import fs from 'node:fs';
14
+ import {Client} from '@modelcontextprotocol/sdk/client/index.js';
15
+ import {InMemoryTransport} from '@modelcontextprotocol/sdk/inMemory.js';
14
16
 
15
17
  // CONSTANTS
16
18
 
@@ -35,19 +37,23 @@ test('mcpPath is /mcp', () => {
35
37
  assert.equal(mcpPath, '/mcp');
36
38
  });
37
39
 
38
- test('createMCPServer registers all 8 tools', () => {
40
+ test('createMCPServer registers all 12 tools', () => {
39
41
  const server = createMCPServer();
40
42
  const toolNames = Object.keys((server as any)._registeredTools);
41
- assert.equal(toolNames.length, 8);
43
+ assert.equal(toolNames.length, 12);
42
44
  assert.deepEqual(toolNames, [
45
+ 'getKilotestOverview',
43
46
  'listReports',
44
47
  'listIssues',
45
48
  'listViolators',
46
49
  'listDiagnoses',
47
50
  'getReport',
48
- 'requestTest',
51
+ 'requestNewTest',
49
52
  'requestRetest',
50
- 'requestFeature'
53
+ 'requestFeature',
54
+ 'orderNewTest',
55
+ 'orderRetest',
56
+ 'awaitTest'
51
57
  ]);
52
58
  });
53
59
 
@@ -56,10 +62,41 @@ test('each tool has a description and a handler function', () => {
56
62
  const tools = (server as any)._registeredTools;
57
63
  for (const [name, tool] of Object.entries(tools) as [string, any][]) {
58
64
  assert.ok(tool.description, `${name} has a description`);
65
+ assert.ok(
66
+ tool.description.includes('front-end quality'),
67
+ `${name} description includes the shared domain context`
68
+ );
59
69
  assert.equal(typeof tool.handler, 'function', `${name} has a handler function`);
60
70
  }
61
71
  });
62
72
 
73
+ test('getKilotestOverview handler returns overview text and records a metric', async () => {
74
+ const server = createMCPServer();
75
+ const countBefore = getToolCallCount('getKilotestOverview');
76
+ const result = await (server as any)._registeredTools.getKilotestOverview.handler({});
77
+ assert.ok(result.content);
78
+ assert.equal(result.content[0].type, 'text');
79
+ assert.ok(result.content[0].text.includes('front-end quality'));
80
+ assert.equal(getToolCallCount('getKilotestOverview'), countBefore + 1);
81
+ });
82
+
83
+ test('client receives instructions, server description, and the overview resource', async () => {
84
+ const server = createMCPServer();
85
+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
86
+ await server.connect(serverTransport);
87
+ const client = new Client({name: 'test-client', version: '1.0.0'});
88
+ await client.connect(clientTransport);
89
+ assert.ok(client.getInstructions()?.includes('front-end quality'));
90
+ assert.ok(client.getServerVersion()?.description?.includes('front-end quality'));
91
+ const resources = await client.listResources();
92
+ assert.deepEqual(resources.resources.map((r: any) => r.uri), ['docs://kilotest/overview']);
93
+ const read = await client.readResource({uri: 'docs://kilotest/overview'});
94
+ assert.equal(read.contents[0]?.mimeType, 'text/plain');
95
+ assert.ok((read.contents[0] as any).text.includes('front-end quality'));
96
+ await client.close();
97
+ await server.close();
98
+ });
99
+
63
100
  test('listReports handler returns content and structuredContent', async () => {
64
101
  const server = createMCPServer();
65
102
  const countBefore = getToolCallCount('listReports');
@@ -125,10 +162,10 @@ test('getReport handler returns content and structuredContent for a valid report
125
162
  assert.equal(getToolCallCount('getReport'), countBefore + 1);
126
163
  });
127
164
 
128
- test('requestTest handler returns content and structuredContent', async () => {
165
+ test('requestNewTest handler returns content and structuredContent', async () => {
129
166
  const server = createMCPServer();
130
- const countBefore = getToolCallCount('requestTest');
131
- const result = await (server as any)._registeredTools.requestTest.handler({
167
+ const countBefore = getToolCallCount('requestNewTest');
168
+ const result = await (server as any)._registeredTools.requestNewTest.handler({
132
169
  description: 'Test Page',
133
170
  URL: 'https://example.com/test',
134
171
  reason: 'Because accessibility matters'
@@ -136,7 +173,7 @@ test('requestTest handler returns content and structuredContent', async () => {
136
173
  assert.ok(result.content);
137
174
  assert.equal(result.content[0].type, 'text');
138
175
  assert.ok(result.structuredContent);
139
- assert.equal(getToolCallCount('requestTest'), countBefore + 1);
176
+ assert.equal(getToolCallCount('requestNewTest'), countBefore + 1);
140
177
  });
141
178
 
142
179
  test('requestRetest handler returns content and structuredContent for a valid report', async () => {
@@ -165,6 +202,60 @@ test('requestFeature handler returns content and structuredContent', async () =>
165
202
  assert.equal(getToolCallCount('requestFeature'), countBefore + 1);
166
203
  });
167
204
 
205
+ test('orderNewTest handler returns content and structuredContent, and enqueues a job', async () => {
206
+ const {jobsPath} = await import('./util.ts');
207
+ const server = createMCPServer();
208
+ const countBefore = getToolCallCount('orderNewTest');
209
+ const result = await (server as any)._registeredTools.orderNewTest.handler({
210
+ description: 'MCP Ordered Page',
211
+ URL: 'https://example.com/mcp-ordered',
212
+ reason: 'Because accessibility matters here'
213
+ });
214
+ assert.ok(result.content);
215
+ assert.equal(result.content[0].type, 'text');
216
+ assert.ok(result.structuredContent);
217
+ assert.equal(getToolCallCount('orderNewTest'), countBefore + 1);
218
+ const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier'];
219
+ assert.ok(reportIdentifier);
220
+ const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`;
221
+ fs.unlinkSync(queuedPath);
222
+ });
223
+
224
+ test('orderRetest handler returns content and structuredContent, and enqueues a job', async () => {
225
+ const {jobsPath} = await import('./util.ts');
226
+ const server = createMCPServer();
227
+ const countBefore = getToolCallCount('orderRetest');
228
+ // 260202T0000-new is the latest report of "Mixed Outcomes Page" (260101T0000-mix is an
229
+ // earlier, superseded report of the same page), so this order is accepted.
230
+ const result = await (server as any)._registeredTools.orderRetest.handler({
231
+ timeStamp: '260202T0000',
232
+ jobID: 'new',
233
+ reason: 'Because the report is obsolete'
234
+ });
235
+ assert.ok(result.content);
236
+ assert.equal(result.content[0].type, 'text');
237
+ assert.ok(result.structuredContent);
238
+ assert.equal(getToolCallCount('orderRetest'), countBefore + 1);
239
+ const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier'];
240
+ assert.ok(reportIdentifier);
241
+ const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`;
242
+ fs.unlinkSync(queuedPath);
243
+ });
244
+
245
+ test('awaitTest handler returns content and structuredContent for an already-completed report', async () => {
246
+ const server = createMCPServer();
247
+ const countBefore = getToolCallCount('awaitTest');
248
+ const result = await (server as any)._registeredTools.awaitTest.handler({
249
+ timeStamp: '260101T0000',
250
+ jobID: 'mix'
251
+ });
252
+ assert.ok(result.content);
253
+ assert.equal(result.content[0].type, 'text');
254
+ assert.ok(result.structuredContent);
255
+ assert.equal(result.structuredContent['response content']['disposition of your wait'].outcome, 'completed');
256
+ assert.equal(getToolCallCount('awaitTest'), countBefore + 1);
257
+ });
258
+
168
259
  test('listIssues handler returns an error for a nonexistent report', async () => {
169
260
  const server = createMCPServer();
170
261
  const result = await (server as any)._registeredTools.listIssues.handler({
@@ -258,6 +349,8 @@ test('handleMCP responds to initialize with server info', async () => {
258
349
  assert.equal(res.statusCode, 200);
259
350
  const result = parseSSEResult(res.body);
260
351
  assert.equal(result.result.serverInfo.name, 'Kilotest');
352
+ assert.ok(result.result.serverInfo.description.includes('front-end quality'));
353
+ assert.ok(result.result.instructions.includes('front-end quality'));
261
354
  assert.equal(result.result.protocolVersion, '2025-06-18');
262
355
  }
263
356
  finally {
@@ -265,7 +358,7 @@ test('handleMCP responds to initialize with server info', async () => {
265
358
  }
266
359
  });
267
360
 
268
- test('handleMCP lists all 8 tools via tools/list', async () => {
361
+ test('handleMCP lists all 12 tools via tools/list', async () => {
269
362
  const server = await startMCPServer();
270
363
  try {
271
364
  const port = server.address().port;
@@ -273,16 +366,20 @@ test('handleMCP lists all 8 tools via tools/list', async () => {
273
366
  assert.equal(res.statusCode, 200);
274
367
  const result = parseSSEResult(res.body);
275
368
  const toolNames = result.result.tools.map((t: any) => t.name);
276
- assert.equal(toolNames.length, 8);
369
+ assert.equal(toolNames.length, 12);
277
370
  assert.deepEqual(toolNames, [
371
+ 'getKilotestOverview',
278
372
  'listReports',
279
373
  'listIssues',
280
374
  'listViolators',
281
375
  'listDiagnoses',
282
376
  'getReport',
283
- 'requestTest',
377
+ 'requestNewTest',
284
378
  'requestRetest',
285
- 'requestFeature'
379
+ 'requestFeature',
380
+ 'orderNewTest',
381
+ 'orderRetest',
382
+ 'awaitTest'
286
383
  ]);
287
384
  }
288
385
  finally {