@jrpool/kilotest 52.0.0 → 57.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +4 -1
- package/.github/workflows/ci.yml +3 -0
- package/README.md +1 -1
- package/api/awaitTest.test.ts +114 -0
- package/api/awaitTest.ts +108 -0
- package/api/listReports.ts +2 -2
- package/api/orderNewTest.test.ts +213 -0
- package/api/orderNewTest.ts +145 -0
- package/api/orderRetest.test.ts +179 -0
- package/api/orderRetest.ts +119 -0
- package/api/requestFeature.test.ts +36 -0
- package/api/requestFeature.ts +26 -4
- package/api/{requestTest.test.ts → requestNewTest.test.ts} +57 -23
- package/api/{requestTest.ts → requestNewTest.ts} +31 -36
- package/api/requestRetest.test.ts +24 -6
- package/api/requestRetest.ts +10 -29
- package/api/routes.test.ts +7 -4
- package/api/routes.ts +45 -7
- package/api/schemas.ts +120 -10
- package/api/util.ts +59 -2
- package/api/version.ts +6 -2
- package/capability.md +9 -6
- package/docs/AI-TOOLS.md +3 -3
- package/docs/SERVICE.md +19 -8
- package/env.example +39 -0
- package/index.test.ts +146 -24
- package/index.ts +129 -36
- package/lhm.plugin.json +1 -1
- package/linkValidation.test.ts +2 -2
- package/llms-full.txt +35 -4
- package/llms.txt +6 -3
- package/mcp.test.ts +109 -12
- package/mcp.ts +146 -26
- package/memory/MEMORY.md +1 -0
- package/memory/plan_issue_111_instant_test.md +198 -0
- package/openapi.yaml +456 -9
- package/package.json +2 -1
- package/routes.test.ts +1 -1
- package/routing.test.ts +2 -2
- package/scripts/generate-openapi.ts +1 -1
- package/server.json +1 -1
- package/sitemap.xml +1 -1
- package/smokeTest.ts +2 -2
- package/util.test.ts +486 -2
- package/util.ts +276 -45
- package/web/listReports/index.html +1 -1
- package/web/listReports/listReports.test.ts +1 -1
- package/web/{requestTest → requestNewTest}/index.ts +10 -1
- package/web/{requestTest/requestTest.test.ts → requestNewTest/requestNewTest.test.ts} +18 -2
- package/web/{requestTestForm → requestNewTestForm}/index.html +1 -1
- package/web/{requestTestForm/requestTestForm.test.ts → requestNewTestForm/requestNewTestForm.test.ts} +6 -6
- package/web/requestRetest/index.ts +9 -0
- package/web/requestRetest/requestRetest.test.ts +14 -0
- package/web/tutorialAI/index.ts +25 -3
- package/web/tutorialAI/tutorialAI.test.ts +32 -0
- package/web/tutorialWeb/index.ts +25 -3
- package/web/tutorialWeb/tutorialWeb.test.ts +32 -0
- /package/web/{requestTest → requestNewTest}/index.html +0 -0
- /package/web/{requestTestForm → requestNewTestForm}/index.ts +0 -0
package/index.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
getObject,
|
|
18
18
|
getPOSTData,
|
|
19
19
|
getReport,
|
|
20
|
+
isAllowedRedirectTarget,
|
|
20
21
|
isAllowedReport,
|
|
21
22
|
isAllowedTarget,
|
|
22
23
|
isReportError,
|
|
@@ -66,8 +67,8 @@ import {answer as renewWCAG} from './web/renewWCAG/index.ts';
|
|
|
66
67
|
import {answer as renewWCAGForm} from './web/renewWCAGForm/index.ts';
|
|
67
68
|
import {answer as requestRetestPage} from './web/requestRetest/index.ts';
|
|
68
69
|
import {answer as requestRetestForm} from './web/requestRetestForm/index.ts';
|
|
69
|
-
import {answer as
|
|
70
|
-
import {answer as
|
|
70
|
+
import {answer as requestNewTestPage} from './web/requestNewTest/index.ts';
|
|
71
|
+
import {answer as requestNewTestForm} from './web/requestNewTestForm/index.ts';
|
|
71
72
|
import {answer as rewindReportsForm} from './web/rewindReportsForm/index.ts';
|
|
72
73
|
import {answer as showHiddenReportsForm} from './web/showHiddenReportsForm/index.ts';
|
|
73
74
|
import {answer as unhideReportForm} from './web/unhideReportForm/index.ts';
|
|
@@ -78,7 +79,10 @@ import {response as listReportsAPI} from './api/listReports.ts';
|
|
|
78
79
|
import {response as listViolatorsAPI} from './api/listViolators.ts';
|
|
79
80
|
import {response as requestFeatureAPI} from './api/requestFeature.ts';
|
|
80
81
|
import {response as requestRetestAPI} from './api/requestRetest.ts';
|
|
81
|
-
import {response as
|
|
82
|
+
import {response as requestNewTestAPI} from './api/requestNewTest.ts';
|
|
83
|
+
import {response as orderNewTestAPI} from './api/orderNewTest.ts';
|
|
84
|
+
import {response as orderRetestAPI} from './api/orderRetest.ts';
|
|
85
|
+
import {response as awaitTestAPI} from './api/awaitTest.ts';
|
|
82
86
|
|
|
83
87
|
// ENVIRONMENT
|
|
84
88
|
|
|
@@ -123,8 +127,8 @@ const answer: {
|
|
|
123
127
|
renewWCAGForm: PageHandler;
|
|
124
128
|
requestRetest: PageHandler;
|
|
125
129
|
requestRetestForm: PageHandler;
|
|
126
|
-
|
|
127
|
-
|
|
130
|
+
requestNewTest: PageHandler;
|
|
131
|
+
requestNewTestForm: PageHandler;
|
|
128
132
|
rewindReportsForm: PageHandler;
|
|
129
133
|
showHiddenReportsForm: PageHandler;
|
|
130
134
|
unhideReportForm: PageHandler;
|
|
@@ -153,8 +157,8 @@ const answer: {
|
|
|
153
157
|
renewWCAGForm,
|
|
154
158
|
requestRetest: requestRetestPage,
|
|
155
159
|
requestRetestForm,
|
|
156
|
-
|
|
157
|
-
|
|
160
|
+
requestNewTest: requestNewTestPage,
|
|
161
|
+
requestNewTestForm,
|
|
158
162
|
rewindReportsForm,
|
|
159
163
|
showHiddenReportsForm,
|
|
160
164
|
unhideReportForm
|
|
@@ -169,7 +173,10 @@ const apiRespond: {
|
|
|
169
173
|
listViolators: ApiResponder;
|
|
170
174
|
requestFeature: ApiResponder;
|
|
171
175
|
requestRetest: ApiResponder;
|
|
172
|
-
|
|
176
|
+
requestNewTest: ApiResponder;
|
|
177
|
+
orderNewTest: ApiResponder;
|
|
178
|
+
orderRetest: ApiResponder;
|
|
179
|
+
awaitTest: ApiResponder;
|
|
173
180
|
} = {
|
|
174
181
|
getReport: getReportAPI,
|
|
175
182
|
listDiagnoses: listDiagnosesAPI,
|
|
@@ -178,7 +185,10 @@ const apiRespond: {
|
|
|
178
185
|
listViolators: listViolatorsAPI,
|
|
179
186
|
requestFeature: requestFeatureAPI,
|
|
180
187
|
requestRetest: requestRetestAPI,
|
|
181
|
-
|
|
188
|
+
requestNewTest: requestNewTestAPI,
|
|
189
|
+
orderNewTest: orderNewTestAPI,
|
|
190
|
+
orderRetest: orderRetestAPI,
|
|
191
|
+
awaitTest: awaitTestAPI
|
|
182
192
|
};
|
|
183
193
|
|
|
184
194
|
// CONSTANTS
|
|
@@ -225,7 +235,7 @@ export const routes = {
|
|
|
225
235
|
'/requestAction.html',
|
|
226
236
|
'/renewWCAG.html',
|
|
227
237
|
'/requestRetest.html/*',
|
|
228
|
-
'/
|
|
238
|
+
'/requestNewTest.html',
|
|
229
239
|
'/tutorialWebComment.html',
|
|
230
240
|
'/unhideReportForm.html',
|
|
231
241
|
'/worker/job',
|
|
@@ -275,12 +285,18 @@ const noDirectGetPages = new Set(['unhideReportForm']);
|
|
|
275
285
|
// engines and other crawlers are still requesting. Redirecting them lets crawlers update
|
|
276
286
|
// their own indexes, rather than leaving them to repeat the same dead request indefinitely.
|
|
277
287
|
// TEMPORARY: retire this redirect after 2027-04-01 once crawlers have re-indexed.
|
|
288
|
+
// requestTest and requestTestForm were renamed to requestNewTest and requestNewTestForm
|
|
289
|
+
// on 2026-09-28 (see the newTest/retest naming-ambiguity fix), to distinguish the
|
|
290
|
+
// new-test-only page from bare "test" language used elsewhere.
|
|
291
|
+
// TEMPORARY: retire this redirect after 2027-03-28 once crawlers have re-indexed.
|
|
278
292
|
const renamedPagePrefixes: Record<string, string> = {
|
|
279
293
|
diagnoses: 'listDiagnoses',
|
|
280
294
|
reportIssue: 'listViolators',
|
|
281
295
|
reportIssues: 'listIssues',
|
|
282
296
|
rules: 'listRules',
|
|
283
|
-
targets: 'listReports'
|
|
297
|
+
targets: 'listReports',
|
|
298
|
+
requestTest: 'requestNewTest',
|
|
299
|
+
requestTestForm: 'requestNewTestForm'
|
|
284
300
|
};
|
|
285
301
|
const jobLock = createLock();
|
|
286
302
|
|
|
@@ -484,26 +500,54 @@ const processJobRequest = async (request: IncomingMessage, response: ServerRespo
|
|
|
484
500
|
if (queuedJobNames.length) {
|
|
485
501
|
const oldestJobName = queuedJobNames[0]!;
|
|
486
502
|
// Get the first one.
|
|
487
|
-
const firstJob = await getObject(path.join(queuePath(), oldestJobName)) as {id: string, sources: {worker: string}, target: {what: string}};
|
|
488
|
-
//
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
//
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
503
|
+
const firstJob = await getObject(path.join(queuePath(), oldestJobName)) as {id: string, sources: {worker: string}, target: {what: string, url: string}};
|
|
504
|
+
// If the job's target is no longer allowed, whether because its host now resolves
|
|
505
|
+
// to a disallowed address or, more likely, because the host now redirects to one
|
|
506
|
+
// (both possible if the target host was reconfigured, e.g. by an attacker, after
|
|
507
|
+
// the request was accepted; a fetch-based check is used, rather than the cheaper
|
|
508
|
+
// DNS-only isAllowedTarget, because this runs only when a job is actually queued,
|
|
509
|
+
// which is rare, and a redirect introduced after acceptance is the likelier attack):
|
|
510
|
+
if (!(await isAllowedRedirectTarget(firstJob.target.url))) {
|
|
511
|
+
console.error(`ERROR: Queued job ${firstJob.id} is on a target this deployment does not allow`);
|
|
512
|
+
// Alert a manager, since this indicates an attempted or accidental SSRF
|
|
513
|
+
// rather than an ordinary usability problem with the target.
|
|
514
|
+
await sendAlert(
|
|
515
|
+
'Kilotest: queued job on disallowed target rejected',
|
|
516
|
+
`Job ${firstJob.id} (${firstJob.target.what}) was never assigned to a worker because its target no longer resolves, directly or via a redirect, to an address this deployment allows (e.g. a private, loopback, or link-local address). The job was reclassified as failed.`
|
|
517
|
+
);
|
|
518
|
+
// Reclassify the job as failed, instead of assigning it or leaving it queued.
|
|
519
|
+
await fs.rename(
|
|
520
|
+
path.join(queuePath(), oldestJobName), path.join(failedPath(), oldestJobName)
|
|
521
|
+
);
|
|
522
|
+
// Respond as if no job were queued, rather than revealing anything about the rejected job.
|
|
523
|
+
response.writeHead(200, {
|
|
524
|
+
'content-type': 'application/json; charset=utf-8'
|
|
525
|
+
});
|
|
526
|
+
response.end(JSON.stringify({}));
|
|
527
|
+
console.log(`${messageStart}but the queued job was on a disallowed target`);
|
|
528
|
+
}
|
|
529
|
+
// Otherwise, i.e. if the job's target is still allowed:
|
|
530
|
+
else {
|
|
531
|
+
// Add the public worker name to the job, in a property Testaro does not read or alter.
|
|
532
|
+
firstJob.sources.worker = workerName;
|
|
533
|
+
console.log(
|
|
534
|
+
`Job ${firstJob.id} (${firstJob.target.what}) is being sent to the worker.`
|
|
535
|
+
);
|
|
536
|
+
// Assign the job to the worker.
|
|
537
|
+
response.writeHead(200, {
|
|
538
|
+
'content-type': 'application/json; charset=utf-8'
|
|
539
|
+
});
|
|
540
|
+
response.end(JSON.stringify(firstJob));
|
|
541
|
+
const messageEnd
|
|
542
|
+
= `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the worker`;
|
|
543
|
+
console.log(`${messageStart}${messageEnd}`);
|
|
544
|
+
// Save the job in the claimed-jobs directory.
|
|
545
|
+
await fs.writeFile(
|
|
546
|
+
path.join(claimedPath(), oldestJobName), getJSON(firstJob)
|
|
547
|
+
);
|
|
548
|
+
// Delete it from the queue.
|
|
549
|
+
await fs.unlink(path.join(queuePath(), oldestJobName));
|
|
550
|
+
}
|
|
507
551
|
}
|
|
508
552
|
// Otherwise, i.e. if no jobs are queued:
|
|
509
553
|
else {
|
|
@@ -898,7 +942,7 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
|
|
|
898
942
|
await serveError({message: 'ERROR: Unreadable request body'}, response, true);
|
|
899
943
|
}
|
|
900
944
|
// If the request is a test request:
|
|
901
|
-
else if (pageName === '
|
|
945
|
+
else if (pageName === 'requestNewTest.html') {
|
|
902
946
|
const {description, url, why} = postData as {description?: string; url: string; why?: string};
|
|
903
947
|
// If the request is valid:
|
|
904
948
|
if (description && isURL(url) && why && await isAllowedTarget(url)) {
|
|
@@ -912,7 +956,7 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
|
|
|
912
956
|
// Serve headers for a response.
|
|
913
957
|
setHeaders('text/html', pathname, 'ultra');
|
|
914
958
|
// Get the answer data.
|
|
915
|
-
const answerData = await answer.
|
|
959
|
+
const answerData = await answer.requestNewTest(description, url, why);
|
|
916
960
|
// If they are valid:
|
|
917
961
|
if (answerData.status === 'ok') {
|
|
918
962
|
// Serve the answer page.
|
|
@@ -1197,15 +1241,26 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
|
|
|
1197
1241
|
// Get the segments of the path after api.
|
|
1198
1242
|
const segments = pathTail.split('/');
|
|
1199
1243
|
// If the service is to receive a test request:
|
|
1200
|
-
if (segments[0] === '
|
|
1244
|
+
if (segments[0] === 'requestNewTest') {
|
|
1201
1245
|
const {description, URL, reason} = postData as {description: string; URL: string; reason: string};
|
|
1202
1246
|
// Get the response body.
|
|
1203
|
-
const responseBody = await apiRespond.
|
|
1204
|
-
await recordPageMetric('apiOperations', '
|
|
1247
|
+
const responseBody = await apiRespond.requestNewTest([description, URL, reason]);
|
|
1248
|
+
await recordPageMetric('apiOperations', 'requestNewTest');
|
|
1205
1249
|
// Send it.
|
|
1206
1250
|
setHeaders('application/json', null, 'ultra');
|
|
1207
1251
|
response.end(JSON.stringify(responseBody));
|
|
1208
1252
|
}
|
|
1253
|
+
// Otherwise, if the request uses the old, pre-rename requestTest path name: redirect
|
|
1254
|
+
// permanently to the new path name. requestTest takes no path parameters (its
|
|
1255
|
+
// arguments are body fields), so there is no further path to preserve, unlike the
|
|
1256
|
+
// renamedPagePrefixes GET-page redirects above. 308 (not 301) is used here because
|
|
1257
|
+
// 301 is defined to let clients rewrite a POST redirect as a GET, which would drop
|
|
1258
|
+
// this request's body; 308 preserves both the method and the body.
|
|
1259
|
+
// TEMPORARY: retire this redirect after 2027-03-28 once callers have updated.
|
|
1260
|
+
else if (segments[0] === 'requestTest') {
|
|
1261
|
+
response.writeHead(308, {Location: '/api/requestNewTest'});
|
|
1262
|
+
response.end();
|
|
1263
|
+
}
|
|
1209
1264
|
// Otherwise, if the service is to receive a retest request:
|
|
1210
1265
|
else if (segments[0] === 'requestRetest') {
|
|
1211
1266
|
const {reason} = postData as {reason: string};
|
|
@@ -1226,6 +1281,44 @@ const handleRequest = async (request: IncomingMessage, response: ServerResponse)
|
|
|
1226
1281
|
setHeaders('application/json', null, 'ultra');
|
|
1227
1282
|
response.end(JSON.stringify(responseBody));
|
|
1228
1283
|
}
|
|
1284
|
+
// Otherwise, if the service is to order an immediate, automatically approved test:
|
|
1285
|
+
else if (segments[0] === 'orderNewTest') {
|
|
1286
|
+
const {description, URL, reason} = postData as {description: string; URL: string; reason: string};
|
|
1287
|
+
// Get the response body.
|
|
1288
|
+
const responseBody = await apiRespond.orderNewTest([description, URL, reason]);
|
|
1289
|
+
await recordPageMetric('apiOperations', 'orderNewTest');
|
|
1290
|
+
// Send it.
|
|
1291
|
+
setHeaders('application/json', null, 'ultra');
|
|
1292
|
+
response.end(JSON.stringify(responseBody));
|
|
1293
|
+
}
|
|
1294
|
+
// Otherwise, if the request uses the old, pre-rename orderTest path name: redirect
|
|
1295
|
+
// permanently to the new path name. orderTest likewise takes no path parameters.
|
|
1296
|
+
// See the requestTest redirect above for why 308 (not 301) is used for these
|
|
1297
|
+
// POST-only API redirects.
|
|
1298
|
+
// TEMPORARY: retire this redirect after 2027-03-28 once callers have updated.
|
|
1299
|
+
else if (segments[0] === 'orderTest') {
|
|
1300
|
+
response.writeHead(308, {Location: '/api/orderNewTest'});
|
|
1301
|
+
response.end();
|
|
1302
|
+
}
|
|
1303
|
+
// Otherwise, if the service is to order an immediate, automatically approved retest:
|
|
1304
|
+
else if (segments[0] === 'orderRetest') {
|
|
1305
|
+
const {reason} = postData as {reason: string};
|
|
1306
|
+
// Get the response body.
|
|
1307
|
+
const responseBody = await apiRespond.orderRetest(segments.slice(1).concat(reason));
|
|
1308
|
+
await recordPageMetric('apiOperations', 'orderRetest');
|
|
1309
|
+
// Send it.
|
|
1310
|
+
setHeaders('application/json', null, 'ultra');
|
|
1311
|
+
response.end(JSON.stringify(responseBody));
|
|
1312
|
+
}
|
|
1313
|
+
// Otherwise, if the service is to wait for an ordered job to complete:
|
|
1314
|
+
else if (segments[0] === 'awaitTest') {
|
|
1315
|
+
// Get the response body.
|
|
1316
|
+
const responseBody = await apiRespond.awaitTest(segments.slice(1));
|
|
1317
|
+
await recordPageMetric('apiOperations', 'awaitTest');
|
|
1318
|
+
// Send it.
|
|
1319
|
+
setHeaders('application/json', null, 'ultra');
|
|
1320
|
+
response.end(JSON.stringify(responseBody));
|
|
1321
|
+
}
|
|
1229
1322
|
// Otherwise, i.e. if the service is invalid:
|
|
1230
1323
|
else {
|
|
1231
1324
|
await serveError(
|
package/lhm.plugin.json
CHANGED
package/linkValidation.test.ts
CHANGED
|
@@ -48,8 +48,8 @@ const htmlRoutes = [
|
|
|
48
48
|
'/listViolators.html',
|
|
49
49
|
'/listRules.html',
|
|
50
50
|
'/manage.html',
|
|
51
|
-
'/
|
|
52
|
-
'/
|
|
51
|
+
'/requestNewTest.html',
|
|
52
|
+
'/requestNewTestForm.html',
|
|
53
53
|
'/requestRetest.html',
|
|
54
54
|
'/requestRetestForm.html',
|
|
55
55
|
'/reannotate.html',
|
package/llms-full.txt
CHANGED
|
@@ -6,7 +6,7 @@ Kilotest is an application that performs ensemble testing of web pages for front
|
|
|
6
6
|
|
|
7
7
|
## What Kilotest does for language models
|
|
8
8
|
|
|
9
|
-
Kilotest is deployed on the public Internet as a service at `https://kilotest.com`. Kilotest can test only web pages that can be accessed
|
|
9
|
+
Kilotest is deployed on the public Internet as a service at `https://kilotest.com`. That deployment of Kilotest can test only web pages that can be accessed by its Testaro worker with a URL. Generally those are pages on the public Internet that are not protected by authentication or other access controls.
|
|
10
10
|
|
|
11
11
|
Without calling any tool, an LLM can, under some conditions, load web pages and use its knowledge to assess successes and failures of front-end quality. But such an assessment is almost always:
|
|
12
12
|
|
|
@@ -63,7 +63,7 @@ Kilotest offers a comprehensive suite of capabilities to users via its web UI:
|
|
|
63
63
|
- Get statistics about issues reported in one report: `https://kilotest.com/listIssues.html/{timeStamp}/{jobID}`
|
|
64
64
|
- Get details about one issue reported in one report: `https://kilotest.com/listViolators.html/{issueID}/{timeStamp}/{jobID}`
|
|
65
65
|
- Get diagnoses by rule engines of rule violations for one HTML element exhibiting one issue in one report: `https://kilotest.com/listDiagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
|
|
66
|
-
- [Recommend the testing of a not-yet-tested page](https://kilotest.com/
|
|
66
|
+
- [Recommend the testing of a not-yet-tested page](https://kilotest.com/requestNewTestForm.html)
|
|
67
67
|
- Recommend the retesting of a previously tested page: `https://kilotest.com/requestRetestForm.html/{timeStamp}/{jobID}`
|
|
68
68
|
- [Get statistics about frequently reported issues across all pages](https://kilotest.com/listTopIssues.html)
|
|
69
69
|
- [Get a list of the operations that Kilotest managers are authorized to request](https://kilotest.com/manage.html)
|
|
@@ -108,10 +108,10 @@ Kilotest offers these tools:
|
|
|
108
108
|
- `timeStamp`: initial segment of report identifier
|
|
109
109
|
- `jobID`: final segment of report identifier
|
|
110
110
|
- source of parameters: response to a `listViolators` request
|
|
111
|
-
- `
|
|
111
|
+
- `requestNewTest`
|
|
112
112
|
- method: `POST`
|
|
113
113
|
- purpose: request testing of one web page about which no report is available
|
|
114
|
-
- path: `/api/
|
|
114
|
+
- path: `/api/requestNewTest`
|
|
115
115
|
- payload properties
|
|
116
116
|
- `description`: description of the page to be tested
|
|
117
117
|
- `URL`: URL of the page to be tested
|
|
@@ -132,6 +132,35 @@ Kilotest offers these tools:
|
|
|
132
132
|
- path: `/api/requestFeature`
|
|
133
133
|
- payload properties
|
|
134
134
|
- `feature`: description and motivation of the feature improvement or the new feature
|
|
135
|
+
- `orderNewTest`
|
|
136
|
+
- method: `POST`
|
|
137
|
+
- purpose: order immediate, automatically approved testing of one web page about which no report is available, instead of waiting for manual approval
|
|
138
|
+
- path: `/api/orderNewTest`
|
|
139
|
+
- payload properties
|
|
140
|
+
- `description`: description of the page to be tested
|
|
141
|
+
- `URL`: URL of the page to be tested
|
|
142
|
+
- `reason`: reason for testing the page
|
|
143
|
+
- response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued
|
|
144
|
+
- `orderRetest`
|
|
145
|
+
- method: `POST`
|
|
146
|
+
- purpose: order immediate, automatically approved retesting of one web page about which a report is available, instead of waiting for manual approval
|
|
147
|
+
- path: `/api/orderRetest/{timeStamp}/{jobID}`
|
|
148
|
+
- parameters
|
|
149
|
+
- `timeStamp`: initial segment of report identifier
|
|
150
|
+
- `jobID`: final segment of report identifier
|
|
151
|
+
- source of parameters: response to a `listIssues` request
|
|
152
|
+
- payload properties
|
|
153
|
+
- `reason`: reason for retesting the page
|
|
154
|
+
- response includes a report identifier (`timeStamp` and `jobID`) once the job is enqueued
|
|
155
|
+
- `awaitTest`
|
|
156
|
+
- method: `POST`
|
|
157
|
+
- purpose: wait for a job ordered via `orderNewTest` or `orderRetest` to complete; blocks until the report is ready, the job fails, or a maximum wait time elapses, then returns
|
|
158
|
+
- path: `/api/awaitTest/{timeStamp}/{jobID}`
|
|
159
|
+
- parameters
|
|
160
|
+
- `timeStamp`: initial segment of the report identifier returned by `orderNewTest` or `orderRetest`
|
|
161
|
+
- `jobID`: final segment of the report identifier returned by `orderNewTest` or `orderRetest`
|
|
162
|
+
- source of parameters: response to an `orderNewTest` or `orderRetest` request
|
|
163
|
+
- caveat: testing typically takes about 2 minutes and occasionally up to 4 minutes; sends no interim notices while waiting
|
|
135
164
|
|
|
136
165
|
## After a test, retest, or feature request
|
|
137
166
|
|
|
@@ -147,6 +176,8 @@ When you request a test or a retest:
|
|
|
147
176
|
- check test completion by calling `listReports` and looking for the page and completion date in the list
|
|
148
177
|
- you advise the user to check in approximately 1 day for the test results on the Kilotest website, starting at `https://kilotest.com/listReports.html`
|
|
149
178
|
|
|
179
|
+
For immediate testing without waiting for manual approval, use `orderNewTest` or `orderRetest` instead of `requestNewTest` or `requestRetest`. Each enqueues the job right away and returns a report identifier along with a question about whether to wait for completion. To wait, call `awaitTest` with that identifier; it blocks until the report is ready, the job fails, or a maximum wait time elapses. To check later instead, call `listIssues` with the same identifier.
|
|
180
|
+
|
|
150
181
|
An [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml) is available.
|
|
151
182
|
|
|
152
183
|
## How to report results honestly to users
|
package/llms.txt
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Kilotest
|
|
2
2
|
|
|
3
|
-
Kilotest helps language models and AI platforms to give thorough, accurate and truthful answers when users ask about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. Kilotest integrates an ensemble of twelve independent rule engines to test
|
|
3
|
+
Kilotest helps language models and AI platforms to give thorough, accurate and truthful answers when users ask about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. Kilotest integrates an ensemble of twelve independent rule engines to test a web page. Kilotest creates a structured report of the results and saves the report for future reference. Kilotest API endpoints and MCP tools can provide multipage, single-page, and element-level test results.
|
|
4
4
|
|
|
5
5
|
## API
|
|
6
6
|
|
|
@@ -9,9 +9,12 @@ Kilotest helps language models and AI platforms to give thorough, accurate and t
|
|
|
9
9
|
- List all elements exhibiting one issue in one report: `/api/listViolators/{issueID}/{timeStamp}/{jobID}` (`GET`)
|
|
10
10
|
- List all diagnoses of one element exhibiting one issue in one report: `/api/listDiagnoses/{catalogIndex}/{issueID}/{timeStamp}/{jobID}` (`GET`)
|
|
11
11
|
- Get one full report in JSON: `/api/getReport/{timeStamp}/{jobID}` (`GET`)
|
|
12
|
-
- Request testing of a web page: `/api/
|
|
12
|
+
- Request testing of a web page: `/api/requestNewTest` (`POST`)
|
|
13
13
|
- Request retesting of a web page: `/api/requestRetest/{timeStamp}/{jobID}` (`POST`)
|
|
14
14
|
- Request adding or improving a feature: `/api/requestFeature` (`POST`)
|
|
15
|
+
- Order immediate, automatically approved testing of a web page: `/api/orderNewTest` (`POST`)
|
|
16
|
+
- Order immediate, automatically approved retesting of a web page: `/api/orderRetest/{timeStamp}/{jobID}` (`POST`)
|
|
17
|
+
- Wait for a job ordered via orderNewTest or orderRetest to complete: `/api/awaitTest/{timeStamp}/{jobID}` (`POST`)
|
|
15
18
|
|
|
16
19
|
## MCP Server
|
|
17
20
|
|
|
@@ -24,7 +27,7 @@ Kilotest helps language models and AI platforms to give thorough, accurate and t
|
|
|
24
27
|
- List all elements exhibiting one issue in one report: `https://kilotest.com/listViolators.html/{issueID}/{timeStamp}/{jobID}`
|
|
25
28
|
- List all diagnoses of one element exhibiting one issue in one report: `https://kilotest.com/listDiagnoses.html/{issueID}/{timeStamp}/{jobID}/{catalogIndex}`
|
|
26
29
|
- Get one full report in JSON: `https://kilotest.com/fullReport.json/{timeStamp}/{jobID}`
|
|
27
|
-
- [Request testing of a web page](https://kilotest.com/
|
|
30
|
+
- [Request testing of a web page](https://kilotest.com/requestNewTestForm.html)
|
|
28
31
|
- Request retesting of a web page: `https://kilotest.com/requestRetestForm.html/{timeStamp}/{jobID}`
|
|
29
32
|
- [List the operations that Kilotest managers are authorized to request](https://kilotest.com/manage.html)
|
|
30
33
|
- [Provide statistics about frequently reported issues across all pages](https://kilotest.com/listTopIssues.html)
|
package/mcp.test.ts
CHANGED
|
@@ -11,6 +11,8 @@ import http from 'node:http';
|
|
|
11
11
|
import {mcpPath, createMCPServer, handleMCP} from './mcp.ts';
|
|
12
12
|
import {metricsPath} from './util.ts';
|
|
13
13
|
import fs from 'node:fs';
|
|
14
|
+
import {Client} from '@modelcontextprotocol/sdk/client/index.js';
|
|
15
|
+
import {InMemoryTransport} from '@modelcontextprotocol/sdk/inMemory.js';
|
|
14
16
|
|
|
15
17
|
// CONSTANTS
|
|
16
18
|
|
|
@@ -35,19 +37,23 @@ test('mcpPath is /mcp', () => {
|
|
|
35
37
|
assert.equal(mcpPath, '/mcp');
|
|
36
38
|
});
|
|
37
39
|
|
|
38
|
-
test('createMCPServer registers all
|
|
40
|
+
test('createMCPServer registers all 12 tools', () => {
|
|
39
41
|
const server = createMCPServer();
|
|
40
42
|
const toolNames = Object.keys((server as any)._registeredTools);
|
|
41
|
-
assert.equal(toolNames.length,
|
|
43
|
+
assert.equal(toolNames.length, 12);
|
|
42
44
|
assert.deepEqual(toolNames, [
|
|
45
|
+
'getKilotestOverview',
|
|
43
46
|
'listReports',
|
|
44
47
|
'listIssues',
|
|
45
48
|
'listViolators',
|
|
46
49
|
'listDiagnoses',
|
|
47
50
|
'getReport',
|
|
48
|
-
'
|
|
51
|
+
'requestNewTest',
|
|
49
52
|
'requestRetest',
|
|
50
|
-
'requestFeature'
|
|
53
|
+
'requestFeature',
|
|
54
|
+
'orderNewTest',
|
|
55
|
+
'orderRetest',
|
|
56
|
+
'awaitTest'
|
|
51
57
|
]);
|
|
52
58
|
});
|
|
53
59
|
|
|
@@ -56,10 +62,41 @@ test('each tool has a description and a handler function', () => {
|
|
|
56
62
|
const tools = (server as any)._registeredTools;
|
|
57
63
|
for (const [name, tool] of Object.entries(tools) as [string, any][]) {
|
|
58
64
|
assert.ok(tool.description, `${name} has a description`);
|
|
65
|
+
assert.ok(
|
|
66
|
+
tool.description.includes('front-end quality'),
|
|
67
|
+
`${name} description includes the shared domain context`
|
|
68
|
+
);
|
|
59
69
|
assert.equal(typeof tool.handler, 'function', `${name} has a handler function`);
|
|
60
70
|
}
|
|
61
71
|
});
|
|
62
72
|
|
|
73
|
+
test('getKilotestOverview handler returns overview text and records a metric', async () => {
|
|
74
|
+
const server = createMCPServer();
|
|
75
|
+
const countBefore = getToolCallCount('getKilotestOverview');
|
|
76
|
+
const result = await (server as any)._registeredTools.getKilotestOverview.handler({});
|
|
77
|
+
assert.ok(result.content);
|
|
78
|
+
assert.equal(result.content[0].type, 'text');
|
|
79
|
+
assert.ok(result.content[0].text.includes('front-end quality'));
|
|
80
|
+
assert.equal(getToolCallCount('getKilotestOverview'), countBefore + 1);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('client receives instructions, server description, and the overview resource', async () => {
|
|
84
|
+
const server = createMCPServer();
|
|
85
|
+
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
|
|
86
|
+
await server.connect(serverTransport);
|
|
87
|
+
const client = new Client({name: 'test-client', version: '1.0.0'});
|
|
88
|
+
await client.connect(clientTransport);
|
|
89
|
+
assert.ok(client.getInstructions()?.includes('front-end quality'));
|
|
90
|
+
assert.ok(client.getServerVersion()?.description?.includes('front-end quality'));
|
|
91
|
+
const resources = await client.listResources();
|
|
92
|
+
assert.deepEqual(resources.resources.map((r: any) => r.uri), ['docs://kilotest/overview']);
|
|
93
|
+
const read = await client.readResource({uri: 'docs://kilotest/overview'});
|
|
94
|
+
assert.equal(read.contents[0]?.mimeType, 'text/plain');
|
|
95
|
+
assert.ok((read.contents[0] as any).text.includes('front-end quality'));
|
|
96
|
+
await client.close();
|
|
97
|
+
await server.close();
|
|
98
|
+
});
|
|
99
|
+
|
|
63
100
|
test('listReports handler returns content and structuredContent', async () => {
|
|
64
101
|
const server = createMCPServer();
|
|
65
102
|
const countBefore = getToolCallCount('listReports');
|
|
@@ -125,10 +162,10 @@ test('getReport handler returns content and structuredContent for a valid report
|
|
|
125
162
|
assert.equal(getToolCallCount('getReport'), countBefore + 1);
|
|
126
163
|
});
|
|
127
164
|
|
|
128
|
-
test('
|
|
165
|
+
test('requestNewTest handler returns content and structuredContent', async () => {
|
|
129
166
|
const server = createMCPServer();
|
|
130
|
-
const countBefore = getToolCallCount('
|
|
131
|
-
const result = await (server as any)._registeredTools.
|
|
167
|
+
const countBefore = getToolCallCount('requestNewTest');
|
|
168
|
+
const result = await (server as any)._registeredTools.requestNewTest.handler({
|
|
132
169
|
description: 'Test Page',
|
|
133
170
|
URL: 'https://example.com/test',
|
|
134
171
|
reason: 'Because accessibility matters'
|
|
@@ -136,7 +173,7 @@ test('requestTest handler returns content and structuredContent', async () => {
|
|
|
136
173
|
assert.ok(result.content);
|
|
137
174
|
assert.equal(result.content[0].type, 'text');
|
|
138
175
|
assert.ok(result.structuredContent);
|
|
139
|
-
assert.equal(getToolCallCount('
|
|
176
|
+
assert.equal(getToolCallCount('requestNewTest'), countBefore + 1);
|
|
140
177
|
});
|
|
141
178
|
|
|
142
179
|
test('requestRetest handler returns content and structuredContent for a valid report', async () => {
|
|
@@ -165,6 +202,60 @@ test('requestFeature handler returns content and structuredContent', async () =>
|
|
|
165
202
|
assert.equal(getToolCallCount('requestFeature'), countBefore + 1);
|
|
166
203
|
});
|
|
167
204
|
|
|
205
|
+
test('orderNewTest handler returns content and structuredContent, and enqueues a job', async () => {
|
|
206
|
+
const {jobsPath} = await import('./util.ts');
|
|
207
|
+
const server = createMCPServer();
|
|
208
|
+
const countBefore = getToolCallCount('orderNewTest');
|
|
209
|
+
const result = await (server as any)._registeredTools.orderNewTest.handler({
|
|
210
|
+
description: 'MCP Ordered Page',
|
|
211
|
+
URL: 'https://example.com/mcp-ordered',
|
|
212
|
+
reason: 'Because accessibility matters here'
|
|
213
|
+
});
|
|
214
|
+
assert.ok(result.content);
|
|
215
|
+
assert.equal(result.content[0].type, 'text');
|
|
216
|
+
assert.ok(result.structuredContent);
|
|
217
|
+
assert.equal(getToolCallCount('orderNewTest'), countBefore + 1);
|
|
218
|
+
const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier'];
|
|
219
|
+
assert.ok(reportIdentifier);
|
|
220
|
+
const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`;
|
|
221
|
+
fs.unlinkSync(queuedPath);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
test('orderRetest handler returns content and structuredContent, and enqueues a job', async () => {
|
|
225
|
+
const {jobsPath} = await import('./util.ts');
|
|
226
|
+
const server = createMCPServer();
|
|
227
|
+
const countBefore = getToolCallCount('orderRetest');
|
|
228
|
+
// 260202T0000-new is the latest report of "Mixed Outcomes Page" (260101T0000-mix is an
|
|
229
|
+
// earlier, superseded report of the same page), so this order is accepted.
|
|
230
|
+
const result = await (server as any)._registeredTools.orderRetest.handler({
|
|
231
|
+
timeStamp: '260202T0000',
|
|
232
|
+
jobID: 'new',
|
|
233
|
+
reason: 'Because the report is obsolete'
|
|
234
|
+
});
|
|
235
|
+
assert.ok(result.content);
|
|
236
|
+
assert.equal(result.content[0].type, 'text');
|
|
237
|
+
assert.ok(result.structuredContent);
|
|
238
|
+
assert.equal(getToolCallCount('orderRetest'), countBefore + 1);
|
|
239
|
+
const reportIdentifier = result.structuredContent['response content']['disposition of your order']['report identifier'];
|
|
240
|
+
assert.ok(reportIdentifier);
|
|
241
|
+
const queuedPath = `${jobsPath()}/queue/${reportIdentifier.timeStamp}-${reportIdentifier.jobID}.json`;
|
|
242
|
+
fs.unlinkSync(queuedPath);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
test('awaitTest handler returns content and structuredContent for an already-completed report', async () => {
|
|
246
|
+
const server = createMCPServer();
|
|
247
|
+
const countBefore = getToolCallCount('awaitTest');
|
|
248
|
+
const result = await (server as any)._registeredTools.awaitTest.handler({
|
|
249
|
+
timeStamp: '260101T0000',
|
|
250
|
+
jobID: 'mix'
|
|
251
|
+
});
|
|
252
|
+
assert.ok(result.content);
|
|
253
|
+
assert.equal(result.content[0].type, 'text');
|
|
254
|
+
assert.ok(result.structuredContent);
|
|
255
|
+
assert.equal(result.structuredContent['response content']['disposition of your wait'].outcome, 'completed');
|
|
256
|
+
assert.equal(getToolCallCount('awaitTest'), countBefore + 1);
|
|
257
|
+
});
|
|
258
|
+
|
|
168
259
|
test('listIssues handler returns an error for a nonexistent report', async () => {
|
|
169
260
|
const server = createMCPServer();
|
|
170
261
|
const result = await (server as any)._registeredTools.listIssues.handler({
|
|
@@ -258,6 +349,8 @@ test('handleMCP responds to initialize with server info', async () => {
|
|
|
258
349
|
assert.equal(res.statusCode, 200);
|
|
259
350
|
const result = parseSSEResult(res.body);
|
|
260
351
|
assert.equal(result.result.serverInfo.name, 'Kilotest');
|
|
352
|
+
assert.ok(result.result.serverInfo.description.includes('front-end quality'));
|
|
353
|
+
assert.ok(result.result.instructions.includes('front-end quality'));
|
|
261
354
|
assert.equal(result.result.protocolVersion, '2025-06-18');
|
|
262
355
|
}
|
|
263
356
|
finally {
|
|
@@ -265,7 +358,7 @@ test('handleMCP responds to initialize with server info', async () => {
|
|
|
265
358
|
}
|
|
266
359
|
});
|
|
267
360
|
|
|
268
|
-
test('handleMCP lists all
|
|
361
|
+
test('handleMCP lists all 12 tools via tools/list', async () => {
|
|
269
362
|
const server = await startMCPServer();
|
|
270
363
|
try {
|
|
271
364
|
const port = server.address().port;
|
|
@@ -273,16 +366,20 @@ test('handleMCP lists all 8 tools via tools/list', async () => {
|
|
|
273
366
|
assert.equal(res.statusCode, 200);
|
|
274
367
|
const result = parseSSEResult(res.body);
|
|
275
368
|
const toolNames = result.result.tools.map((t: any) => t.name);
|
|
276
|
-
assert.equal(toolNames.length,
|
|
369
|
+
assert.equal(toolNames.length, 12);
|
|
277
370
|
assert.deepEqual(toolNames, [
|
|
371
|
+
'getKilotestOverview',
|
|
278
372
|
'listReports',
|
|
279
373
|
'listIssues',
|
|
280
374
|
'listViolators',
|
|
281
375
|
'listDiagnoses',
|
|
282
376
|
'getReport',
|
|
283
|
-
'
|
|
377
|
+
'requestNewTest',
|
|
284
378
|
'requestRetest',
|
|
285
|
-
'requestFeature'
|
|
379
|
+
'requestFeature',
|
|
380
|
+
'orderNewTest',
|
|
381
|
+
'orderRetest',
|
|
382
|
+
'awaitTest'
|
|
286
383
|
]);
|
|
287
384
|
}
|
|
288
385
|
finally {
|