@jrpool/kilotest 46.0.8 → 46.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +4 -1
- package/.github/workflows/{smoke-test.yml → ci.yml} +8 -2
- package/README.md +14 -8
- package/alerts.test.js +137 -0
- package/api/listDiagnoses.test.js +44 -0
- package/api/listIssues.js +1 -1
- package/api/listIssues.test.js +10 -0
- package/api/listReports.test.js +6 -2
- package/api/listViolators.test.js +35 -0
- package/api/requestTest.test.js +7 -0
- package/api/routes.test.js +63 -0
- package/api/util.js +9 -11
- package/api/util.test.js +68 -2
- package/apiIntegration.test.js +13 -3
- package/docs/AI-TOOLS.md +41 -158
- package/docs/{Smoke Test.json → CI.json} +4 -3
- package/index.html +3 -3
- package/index.js +64 -77
- package/index.test.js +1152 -0
- package/indexBalances.test.js +188 -0
- package/indexFallback.test.js +33 -0
- package/indexServe.test.js +59 -0
- package/indexWorkers.test.js +79 -0
- package/lhm.plugin.json +3218 -0
- package/llms-full.txt +6 -7
- package/llms.txt +1 -1
- package/mcp.js +2 -2
- package/mcp.test.js +281 -0
- package/openapi.yaml +1 -1
- package/package.json +10 -5
- package/routing.test.js +13 -3
- package/scripts/generate-openapi.js +1 -1
- package/test/fixtures/buildFixtures.js +83 -0
- package/test/fixtures/db/reports/260101T0008-mul.json +137 -0
- package/test/fixtures/db/reports/260101T0009-brd.json +147 -0
- package/test/setup.cjs +14 -0
- package/util.js +16 -6
- package/util.test.js +886 -3
- package/web/ai0BalanceForm/ai0BalanceForm.test.js +88 -0
- package/web/expungeReportsForm/expungeReportsForm.test.js +114 -0
- package/web/hideReportForm/hideReportForm.test.js +82 -0
- package/web/listDiagnoses/listDiagnoses.test.js +160 -0
- package/web/listIssues/listIssues.test.js +109 -0
- package/web/listReports/listReports.test.js +139 -3
- package/web/listRules/listRules.test.js +103 -0
- package/web/listTopIssues/listTopIssues.test.js +126 -0
- package/web/listViolators/index.js +1 -2
- package/web/listViolators/listViolators.test.js +256 -0
- package/web/pruneReportsForm/pruneReportsForm.test.js +86 -0
- package/web/reannotate/reannotate.test.js +144 -0
- package/web/reannotateForm/reannotateForm.test.js +134 -0
- package/web/renewWCAG/renewWCAG.test.js +124 -0
- package/web/requestRetest/requestRetest.test.js +119 -0
- package/web/requestRetestForm/requestRetestForm.test.js +62 -0
- package/web/requestTest/requestTest.test.js +115 -0
- package/web/requestTestForm/requestTestForm.test.js +40 -0
- package/web/rewindReportsForm/rewindReportsForm.test.js +86 -0
- package/web/tutorial/index.html +3 -3
- package/web/tutorial/tutorial.test.js +97 -0
- package/web/unhideReportForm/unhideReportForm.test.js +135 -0
package/.devin/config.local.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
name:
|
|
1
|
+
name: CI
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
pull_request:
|
|
5
5
|
branches: [main]
|
|
6
6
|
|
|
7
7
|
jobs:
|
|
8
|
-
|
|
8
|
+
ci:
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
10
|
permissions:
|
|
11
11
|
contents: read
|
|
@@ -22,5 +22,11 @@ jobs:
|
|
|
22
22
|
- name: Install dependencies
|
|
23
23
|
run: npm ci
|
|
24
24
|
|
|
25
|
+
- name: Run lint
|
|
26
|
+
run: npm run lint
|
|
27
|
+
|
|
28
|
+
- name: Run tests with coverage
|
|
29
|
+
run: npm test
|
|
30
|
+
|
|
25
31
|
- name: Run smoke test against deployed server
|
|
26
32
|
run: node smokeTest.js
|
package/README.md
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
# Kilotest
|
|
2
2
|
|
|
3
|
+
[](https://www.npmjs.com/package/@jrpool/kilotest)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](https://registry.modelcontextprotocol.io/v0/servers?search=kilotest)
|
|
6
|
+
[](https://github.com/jrpool/kilotest/stargazers)
|
|
7
|
+
|
|
3
8
|
An ensemble testing and reporting service for front-end web quality
|
|
4
9
|
|
|
5
10
|
## Features
|
|
6
11
|
|
|
7
|
-
This application uses an ensemble of
|
|
12
|
+
This application uses an ensemble of 12 rule engines to test public web pages for front-end quality (i.e. accessibility, usability, and standards conformity).
|
|
8
13
|
|
|
9
14
|
The testing paradigm employed by Kilotest is discussed in these papers:
|
|
10
15
|
|
|
@@ -24,7 +29,7 @@ An active production instance of Kilotest may require multiple testing agents to
|
|
|
24
29
|
|
|
25
30
|
In the steps below, hosts `T` and `K` may be the same host or two different hosts. Host `T` can be a Debian stable, Ubuntu LTS, Windows, or macOS host. Host `K` can be any server host that can run the latest LTS version of Node.js. If hosts `T` and `K` differ, then they must be open to `https` traffic and host `K` must permit `https` requests from host `T`.
|
|
26
31
|
|
|
27
|
-
1. Clone the [Testaro project](https://github.com/
|
|
32
|
+
1. Clone the [Testaro project](https://github.com/YRA-Tech/testaro) into a new directory on host `T`.
|
|
28
33
|
1. In that directory, install the Testaro dependencies: `npm install`.
|
|
29
34
|
1. Update the Testaro dependencies and rebuild: `npm run deps`.
|
|
30
35
|
1. Clone the Kilotest repository into a new directory on host `K`.
|
|
@@ -43,20 +48,21 @@ Contributions are welcome! You can use GitHub issues to initiate discussions and
|
|
|
43
48
|
|
|
44
49
|
## Making Kilotest a service
|
|
45
50
|
|
|
46
|
-
See the `SERVICE.md` file for instructions on how to make Kilotest a service.
|
|
51
|
+
See the [`SERVICE.md`](docs/SERVICE.md) file for instructions on how to make Kilotest a service.
|
|
47
52
|
|
|
48
53
|
## Using Kilotest as an AI tool
|
|
49
54
|
|
|
50
55
|
Kilotest is not only a web application but also an API and an MCP server. AI platforms and models can use Kilotest via those interfaces.
|
|
51
56
|
|
|
52
|
-
The URL for models using the MCP server is `
|
|
57
|
+
The URL for models using the MCP server is `https://kilotest.com/mcp`. The MCP server is listed at these services:
|
|
53
58
|
|
|
54
59
|
- [Smithery](https://smithery.ai/servers/pool/kilotest)
|
|
55
60
|
- [Glama](https://glama.ai/mcp/connectors/com.kilotest/kilotest)
|
|
56
61
|
|
|
57
62
|
There is additional documentation at:
|
|
58
63
|
|
|
59
|
-
- [
|
|
60
|
-
- [
|
|
61
|
-
- [llms
|
|
62
|
-
- [
|
|
64
|
+
- [QAI](https://kilotest.com/qai): a tutorial for allowing AI assistants to use Kilotest
|
|
65
|
+
- [AI-TOOLS.md](docs/AI-TOOLS.md): how Kilotest implements the MCP protocol
|
|
66
|
+
- [llms.txt](llms.txt): basic documentation for LLMs
|
|
67
|
+
- [llms-full.txt](llms-full.txt): comprehensive documentation for LLMs
|
|
68
|
+
- [lhm.plugin.json](lhm.plugin.json): documentation for registration with the LobeHub MCP Market
|
package/alerts.test.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/*
|
|
2
|
+
alerts.test.js
|
|
3
|
+
Tests for alerts.js, covering the success, failure, error, timeout, and unconfigured paths.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// IMPORTS
|
|
7
|
+
|
|
8
|
+
const {test, before, after} = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
const https = require('node:https');
|
|
11
|
+
const {EventEmitter} = require('node:events');
|
|
12
|
+
|
|
13
|
+
// SETUP AND TEARDOWN
|
|
14
|
+
|
|
15
|
+
const originalRequest = https.request;
|
|
16
|
+
const savedEnv = {};
|
|
17
|
+
|
|
18
|
+
before(() => {
|
|
19
|
+
for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) {
|
|
20
|
+
savedEnv[key] = process.env[key];
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
after(() => {
|
|
25
|
+
https.request = originalRequest;
|
|
26
|
+
for (const key of Object.keys(savedEnv)) {
|
|
27
|
+
if (savedEnv[key] !== undefined) {
|
|
28
|
+
process.env[key] = savedEnv[key];
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
delete process.env[key];
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
delete require.cache[require.resolve('./alerts')];
|
|
35
|
+
require('./alerts');
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// HELPER
|
|
39
|
+
|
|
40
|
+
// Replaces https.request with a function that returns a fake request and invokes the callback with a fake response. The mockHandler is called on next tick so that event listeners are attached first.
|
|
41
|
+
const mockRequest = mockHandler => {
|
|
42
|
+
https.request = (options, callback) => {
|
|
43
|
+
const req = new EventEmitter();
|
|
44
|
+
req.write = () => {};
|
|
45
|
+
req.end = () => {};
|
|
46
|
+
req.destroy = () => {
|
|
47
|
+
req.emit('error', new Error('Socket destroyed'));
|
|
48
|
+
};
|
|
49
|
+
req.setTimeout = (ms, fn) => {
|
|
50
|
+
process.nextTick(fn);
|
|
51
|
+
};
|
|
52
|
+
const res = new EventEmitter();
|
|
53
|
+
res.statusCode = 200;
|
|
54
|
+
process.nextTick(() => mockHandler(req, res, callback));
|
|
55
|
+
return req;
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
// Sets the alert configuration env vars so that sendAlert enters the configured path.
|
|
60
|
+
const setAlertConfig = () => {
|
|
61
|
+
process.env.MANAGER_EMAIL = 'manager@example.com';
|
|
62
|
+
process.env.ALERT_API_HOST = 'alert.example.com';
|
|
63
|
+
process.env.ALERT_API_PATH = '/send';
|
|
64
|
+
process.env.ALERT_API_KEY = 'test-key';
|
|
65
|
+
process.env.ALERT_FROM = 'kilotest@example.com';
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
// Clears the alert configuration env vars so that sendAlert enters the unconfigured path.
|
|
69
|
+
const clearAlertConfig = () => {
|
|
70
|
+
for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) {
|
|
71
|
+
delete process.env[key];
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// Loads alerts.js fresh so it picks up the current env vars and https.request.
|
|
76
|
+
const loadAlerts = () => {
|
|
77
|
+
delete require.cache[require.resolve('./alerts')];
|
|
78
|
+
return require('./alerts');
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
// TESTS
|
|
82
|
+
|
|
83
|
+
test('sendAlert resolves without sending when configuration is incomplete', async () => {
|
|
84
|
+
clearAlertConfig();
|
|
85
|
+
const {sendAlert} = loadAlerts();
|
|
86
|
+
await sendAlert('Test', 'Body');
|
|
87
|
+
assert.ok(true);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('sendAlert logs success when the API responds with a 2xx status', async () => {
|
|
91
|
+
setAlertConfig();
|
|
92
|
+
mockRequest((req, res, callback) => {
|
|
93
|
+
res.statusCode = 200;
|
|
94
|
+
callback(res);
|
|
95
|
+
res.emit('end');
|
|
96
|
+
});
|
|
97
|
+
const {sendAlert} = loadAlerts();
|
|
98
|
+
await sendAlert('Test Success', 'Body');
|
|
99
|
+
https.request = originalRequest;
|
|
100
|
+
assert.ok(true);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('sendAlert logs an error when the API responds with a non-2xx status', async () => {
|
|
104
|
+
setAlertConfig();
|
|
105
|
+
mockRequest((req, res, callback) => {
|
|
106
|
+
res.statusCode = 500;
|
|
107
|
+
callback(res);
|
|
108
|
+
res.emit('data', 'Server error');
|
|
109
|
+
res.emit('end');
|
|
110
|
+
});
|
|
111
|
+
const {sendAlert} = loadAlerts();
|
|
112
|
+
await sendAlert('Test Failure', 'Body');
|
|
113
|
+
https.request = originalRequest;
|
|
114
|
+
assert.ok(true);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test('sendAlert logs an error when the request errors', async () => {
|
|
118
|
+
setAlertConfig();
|
|
119
|
+
mockRequest((req) => {
|
|
120
|
+
req.emit('error', new Error('Connection refused'));
|
|
121
|
+
});
|
|
122
|
+
const {sendAlert} = loadAlerts();
|
|
123
|
+
await sendAlert('Test Error', 'Body');
|
|
124
|
+
https.request = originalRequest;
|
|
125
|
+
assert.ok(true);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test('sendAlert logs an error when the request times out', async () => {
|
|
129
|
+
setAlertConfig();
|
|
130
|
+
mockRequest(() => {
|
|
131
|
+
// Do nothing; the setTimeout mock will fire and call req.destroy().
|
|
132
|
+
});
|
|
133
|
+
const {sendAlert} = loadAlerts();
|
|
134
|
+
await sendAlert('Test Timeout', 'Body');
|
|
135
|
+
https.request = originalRequest;
|
|
136
|
+
assert.ok(true);
|
|
137
|
+
});
|
|
@@ -86,3 +86,47 @@ test('listDiagnoses returns an error for a nonexistent report', async () => {
|
|
|
86
86
|
const basics = body['response content']['basics about the report'];
|
|
87
87
|
assert.ok(basics.error);
|
|
88
88
|
});
|
|
89
|
+
|
|
90
|
+
test('listDiagnoses returns guideline layer for an issue with a short WCAG code', async () => {
|
|
91
|
+
const body = await response(['0', 'duplicateID', '260101T0009', 'brd']);
|
|
92
|
+
const issueBasics = body['response content']['basics about the issue'];
|
|
93
|
+
assert.equal(issueBasics['related WCAG standard'].layer, 'guideline');
|
|
94
|
+
assert.equal(issueBasics['related WCAG standard'].identifier, '4.1');
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test('listDiagnoses returns null tag name and inner text for a catalog item missing them', async () => {
|
|
98
|
+
const body = await response(['1', 'duplicateID', '260101T0009', 'brd']);
|
|
99
|
+
const elementBasics = body['response content']['basics about the element'];
|
|
100
|
+
assert.equal(elementBasics['tag name'], null);
|
|
101
|
+
assert.equal(elementBasics['inner text'], null);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('listDiagnoses returns null rule ID when ruleID equals what', async () => {
|
|
105
|
+
const body = await response(['0', 'duplicateID', '260101T0009', 'brd']);
|
|
106
|
+
const diagnoses = body['response content']['diagnoses of how the element exhibited the issue'];
|
|
107
|
+
const axeDiagnosis = diagnoses.find(d => d['description of the violation'] === 'r99');
|
|
108
|
+
assert.ok(axeDiagnosis);
|
|
109
|
+
assert.equal(axeDiagnosis['identifier of the violated rule'], null);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('listDiagnoses defaults count to 1 when count is missing', async () => {
|
|
113
|
+
const body = await response(['1', 'duplicateID', '260101T0009', 'brd']);
|
|
114
|
+
const diagnoses = body['response content']['diagnoses of how the element exhibited the issue'];
|
|
115
|
+
const ibmDiagnosis = diagnoses.find(d => d['description of the violation'] === 'Element has no role');
|
|
116
|
+
assert.ok(ibmDiagnosis);
|
|
117
|
+
assert.equal(ibmDiagnosis['count of violations of the rule by the element'], 1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test('listDiagnoses returns null for missing start tag, XPath, and bounding box', async () => {
|
|
121
|
+
const body = await response(['2', 'duplicateID', '260101T0009', 'brd']);
|
|
122
|
+
const elementDetails = body['response content']['details about the element'];
|
|
123
|
+
assert.equal(elementDetails['start tag'], null);
|
|
124
|
+
assert.equal(elementDetails['XPath'], null);
|
|
125
|
+
assert.equal(elementDetails['x, y, width, and height of bounding box'], null);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test('listDiagnoses returns issue error for a falsy issue ID', async () => {
|
|
129
|
+
const body = await response(['0', '', '260101T0009', 'brd']);
|
|
130
|
+
const issueBasics = body['response content']['basics about the issue'];
|
|
131
|
+
assert.ok(issueBasics.error);
|
|
132
|
+
});
|
package/api/listIssues.js
CHANGED
|
@@ -111,7 +111,7 @@ exports.response = async args => {
|
|
|
111
111
|
'name': getRuleEngineFacts(ruleEngineID).name,
|
|
112
112
|
'reason for failure': reason
|
|
113
113
|
}));
|
|
114
|
-
const sortedPreventionFacts = objectSort(preventionFacts
|
|
114
|
+
const sortedPreventionFacts = objectSort(preventionFacts, 'name', 'alpha');
|
|
115
115
|
// Initialize the counts of issues by weight.
|
|
116
116
|
const weightCounts = [0, 0, 0, 0];
|
|
117
117
|
// For each issue:
|
package/api/listIssues.test.js
CHANGED
|
@@ -96,3 +96,13 @@ test('listIssues returns an error for a nonexistent report', async () => {
|
|
|
96
96
|
const basics = body['response content']['basics about the report'];
|
|
97
97
|
assert.ok(basics.error);
|
|
98
98
|
});
|
|
99
|
+
|
|
100
|
+
test('listIssues handles instances with missing issueID and null instances without error', async () => {
|
|
101
|
+
const body = await response(['260101T0009', 'brd']);
|
|
102
|
+
const issues = body['response content']['basics about all issues reported in the report'];
|
|
103
|
+
// The brd fixture has one instance with duplicateID (a valid issue) and one with no issueID (skipped).
|
|
104
|
+
// The alfa act has no instances array, so it should be handled as empty.
|
|
105
|
+
assert.ok(issues.length >= 1);
|
|
106
|
+
const ids = issues.map(i => i.identifier);
|
|
107
|
+
assert.ok(ids.includes('duplicateID'));
|
|
108
|
+
});
|
package/api/listReports.test.js
CHANGED
|
@@ -19,6 +19,8 @@ const fixtureIds = [
|
|
|
19
19
|
['260101T0002', 'no'],
|
|
20
20
|
['260101T0005', 'emp'],
|
|
21
21
|
['260101T0006', 'prv'],
|
|
22
|
+
['260101T0008', 'mul'],
|
|
23
|
+
['260101T0009', 'brd'],
|
|
22
24
|
['260202T0000', 'new']
|
|
23
25
|
];
|
|
24
26
|
|
|
@@ -41,10 +43,10 @@ after(() => {
|
|
|
41
43
|
|
|
42
44
|
// TESTS
|
|
43
45
|
|
|
44
|
-
test('listReports returns basics about all
|
|
46
|
+
test('listReports returns basics about all 8 non-hidden reports', async () => {
|
|
45
47
|
const body = await response();
|
|
46
48
|
const reportsBasics = body['response content']['basics about all available reports'];
|
|
47
|
-
assert.equal(reportsBasics.length,
|
|
49
|
+
assert.equal(reportsBasics.length, 8);
|
|
48
50
|
const ids = reportsBasics.map(b => b.identifier).sort();
|
|
49
51
|
const expectedIds = fixtureIds.map(([ts, jid]) => `${ts}-${jid}`).sort();
|
|
50
52
|
assert.deepEqual(ids, expectedIds);
|
|
@@ -66,9 +68,11 @@ test('listReports sorts reports by page description and then by completion time'
|
|
|
66
68
|
descriptions,
|
|
67
69
|
[
|
|
68
70
|
'All CantTell Page',
|
|
71
|
+
'Branch Coverage Page',
|
|
69
72
|
'Empty Results Page',
|
|
70
73
|
'Mixed Outcomes Page',
|
|
71
74
|
'Mixed Outcomes Page',
|
|
75
|
+
'Multi Violator Page',
|
|
72
76
|
'No Outcomes Page',
|
|
73
77
|
'Prevented Page'
|
|
74
78
|
]
|
|
@@ -71,3 +71,38 @@ test('listViolators includes reporter facts for the issue', async () => {
|
|
|
71
71
|
const names = reporters.map(r => r.name).sort();
|
|
72
72
|
assert.deepEqual(names, ['Alfa', 'Axe']);
|
|
73
73
|
});
|
|
74
|
+
|
|
75
|
+
test('listViolators sorts by catalogIndex when two violators have the same reporter count', async () => {
|
|
76
|
+
const body = await response(['linkNoText', '260101T0008', 'mul']);
|
|
77
|
+
const violators = body['response content']['basics about all elements exhibiting the issue'];
|
|
78
|
+
assert.equal(violators.length, 3);
|
|
79
|
+
// The violator with 2 reporters (catalogIndex 2) sorts first.
|
|
80
|
+
assert.equal(violators[0].identifier, '2');
|
|
81
|
+
assert.equal(violators[0]['count of rule engines reporting that the element exhibited the issue'], 2);
|
|
82
|
+
// The two violators with 1 reporter each are sorted by catalogIndex ascending.
|
|
83
|
+
assert.equal(violators[1].identifier, '0');
|
|
84
|
+
assert.equal(violators[1]['count of rule engines reporting that the element exhibited the issue'], 1);
|
|
85
|
+
assert.equal(violators[2].identifier, '1');
|
|
86
|
+
assert.equal(violators[2]['count of rule engines reporting that the element exhibited the issue'], 1);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('listViolators returns guideline layer for an issue with a short WCAG code', async () => {
|
|
90
|
+
const body = await response(['duplicateID', '260101T0009', 'brd']);
|
|
91
|
+
const issueBasics = body['response content']['basics about the issue'];
|
|
92
|
+
assert.equal(issueBasics['related WCAG standard'].layer, 'guideline');
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('listViolators returns null tag name and text for a violator not in the catalog', async () => {
|
|
96
|
+
const body = await response(['duplicateID', '260101T0009', 'brd']);
|
|
97
|
+
const violators = body['response content']['basics about all elements exhibiting the issue'];
|
|
98
|
+
const orphan = violators.find(v => v.identifier === '3');
|
|
99
|
+
assert.ok(orphan);
|
|
100
|
+
assert.equal(orphan['tag name'], null);
|
|
101
|
+
assert.equal(orphan['inner text'], null);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('listViolators returns an error for a falsy issue ID', async () => {
|
|
105
|
+
const body = await response(['', '260101T0009', 'brd']);
|
|
106
|
+
const issueBasics = body['response content']['basics about the issue'];
|
|
107
|
+
assert.ok(issueBasics.error);
|
|
108
|
+
});
|
package/api/requestTest.test.js
CHANGED
|
@@ -57,6 +57,13 @@ test('requestTest rejects a URL shorter than 12 characters', async () => {
|
|
|
57
57
|
assert.equal(processTestRequestCalls.length, 0);
|
|
58
58
|
});
|
|
59
59
|
|
|
60
|
+
test('requestTest rejects a syntactically invalid URL with the correct length', async () => {
|
|
61
|
+
const body = await response(['Test Page', 'not-a-valid-url', 'A reason that is long enough.']);
|
|
62
|
+
const details = body['response content']['details about your request'];
|
|
63
|
+
assert.ok(details.error.includes('invalid URL'));
|
|
64
|
+
assert.equal(processTestRequestCalls.length, 0);
|
|
65
|
+
});
|
|
66
|
+
|
|
60
67
|
test('requestTest rejects an already-tested page', async () => {
|
|
61
68
|
const body = await response(['Mixed Outcomes Page', 'https://example.com/mixed', 'A reason that is long enough.']);
|
|
62
69
|
const details = body['response content']['details about your request'];
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/*
|
|
2
|
+
routes.test.js
|
|
3
|
+
Tests for api/routes.js, verifying the structure and completeness of the OpenAPI route metadata.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// IMPORTS
|
|
7
|
+
|
|
8
|
+
const {test} = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
const routes = require('./routes');
|
|
11
|
+
|
|
12
|
+
// TESTS
|
|
13
|
+
|
|
14
|
+
test('routes exports an array with 8 operations', () => {
|
|
15
|
+
assert.ok(Array.isArray(routes));
|
|
16
|
+
assert.equal(routes.length, 8);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('routes includes all expected operation IDs', () => {
|
|
20
|
+
const ids = routes.map(r => r.operationId).sort();
|
|
21
|
+
assert.deepEqual(ids, [
|
|
22
|
+
'getReport',
|
|
23
|
+
'listDiagnoses',
|
|
24
|
+
'listIssues',
|
|
25
|
+
'listReports',
|
|
26
|
+
'listViolators',
|
|
27
|
+
'requestFeature',
|
|
28
|
+
'requestRetest',
|
|
29
|
+
'requestTest'
|
|
30
|
+
]);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('every route has a method, path, summary, and responseSchema', () => {
|
|
34
|
+
for (const route of routes) {
|
|
35
|
+
assert.ok(route.operationId, `route missing operationId: ${JSON.stringify(route)}`);
|
|
36
|
+
assert.ok(route.method, `route ${route.operationId} missing method`);
|
|
37
|
+
assert.ok(route.path, `route ${route.operationId} missing path`);
|
|
38
|
+
assert.ok(route.summary, `route ${route.operationId} missing summary`);
|
|
39
|
+
assert.ok(route.responseSchema, `route ${route.operationId} missing responseSchema`);
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('GET routes have pathParamsSchema when the path has parameters', () => {
|
|
44
|
+
for (const route of routes) {
|
|
45
|
+
if (route.method === 'get' && route.path.includes('{')) {
|
|
46
|
+
assert.ok(route.pathParamsSchema, `route ${route.operationId} missing pathParamsSchema`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('POST routes have bodySchema', () => {
|
|
52
|
+
for (const route of routes) {
|
|
53
|
+
if (route.method === 'post') {
|
|
54
|
+
assert.ok(route.bodySchema, `route ${route.operationId} missing bodySchema`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('route paths start with /api/', () => {
|
|
60
|
+
for (const route of routes) {
|
|
61
|
+
assert.ok(route.path.startsWith('/api/'), `route ${route.operationId} path does not start with /api/`);
|
|
62
|
+
}
|
|
63
|
+
});
|
package/api/util.js
CHANGED
|
@@ -35,7 +35,7 @@ exports.getResponseMetadata = () => ({
|
|
|
35
35
|
exports.getToolsFacts = () => ({
|
|
36
36
|
'name': 'Kilotest',
|
|
37
37
|
'description': {
|
|
38
|
-
'what Kilotest does': 'Kilotest tools generate and make available findings about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. A Kilotest job generates findings by using Testaro to test a page against more than a thousand rules defined by an ensemble of
|
|
38
|
+
'what Kilotest does': 'Kilotest tools generate and make available findings about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. A Kilotest job generates findings by using Testaro to test a page against more than a thousand rules defined by an ensemble of twelve rule engines. Testaro produces a report of the job. The report describes violations of the rules. Kilotest uses Testilo to enhance the report with a classification of the rule violations into about 300 issues. Kilotest makes facts about the issues and the violations retrievable at four levels of granularity.',
|
|
39
39
|
'how to retrieve findings': {
|
|
40
40
|
'level 1': 'Use the listReports tool to get a list of available reports.',
|
|
41
41
|
'level 2': 'Use the listIssues tool to get a list of issues in one report.',
|
|
@@ -67,7 +67,7 @@ exports.getRuleEnginesFacts = ruleEngineIDSet => {
|
|
|
67
67
|
objectSort(ruleEnginesFacts, 'name', 'alpha');
|
|
68
68
|
return ruleEnginesFacts;
|
|
69
69
|
};
|
|
70
|
-
// Returns the basics about a report
|
|
70
|
+
// Returns the basics about a report.
|
|
71
71
|
exports.getReportBasics = async (timeStamp, jobID) => {
|
|
72
72
|
// Get the creation time of the report.
|
|
73
73
|
const reportStats = await getReportStats(timeStamp, jobID);
|
|
@@ -79,24 +79,22 @@ exports.getReportBasics = async (timeStamp, jobID) => {
|
|
|
79
79
|
error: `Report ${timeStamp}-${jobID} could not be retrieved.`
|
|
80
80
|
};
|
|
81
81
|
}
|
|
82
|
-
//
|
|
82
|
+
// Otherwise, i.e. if it exists, get an extract of the report.
|
|
83
83
|
const reportExtract = await getReportExtract(timeStamp, jobID);
|
|
84
|
-
// If no such report exists:
|
|
85
|
-
if (reportExtract.error) {
|
|
86
|
-
// Return this.
|
|
87
|
-
return reportExtract;
|
|
88
|
-
}
|
|
89
84
|
const {url, what} = reportExtract;
|
|
90
85
|
const latestReportExtracts = await getLatestReportExtracts();
|
|
91
86
|
// Otherwise, i.e. if the report exists, get whether this report has been superseded.
|
|
92
87
|
const isSuperseded = latestReportExtracts
|
|
93
88
|
.every(extract => extract.timeStamp !== timeStamp || extract.jobID !== jobID);
|
|
94
|
-
|
|
89
|
+
// Get the completion time from the report content, not the file system birth time,
|
|
90
|
+
// because the birth time depends on when the file was created on the file system
|
|
91
|
+
// and does not reflect the actual report completion time.
|
|
92
|
+
const {reportTime} = reportExtract;
|
|
95
93
|
// Get the basics about the report.
|
|
96
94
|
const basics = {
|
|
97
95
|
identifier: `${timeStamp}-${jobID}`,
|
|
98
|
-
'completion date and time': reportTime
|
|
99
|
-
'days since the report was completed': getAgoDays(reportTime),
|
|
96
|
+
'completion date and time': reportTime,
|
|
97
|
+
'days since the report was completed': getAgoDays(new Date(reportTime)),
|
|
100
98
|
'tested web page': {
|
|
101
99
|
description: what,
|
|
102
100
|
URL: url
|
package/api/util.test.js
CHANGED
|
@@ -7,9 +7,12 @@
|
|
|
7
7
|
|
|
8
8
|
const {test} = require('node:test');
|
|
9
9
|
const assert = require('node:assert/strict');
|
|
10
|
-
const
|
|
10
|
+
const path = require('node:path');
|
|
11
|
+
const {getReportBasics, getResponseMetadata, getRuleEngineFacts, getRuleEnginesFacts, getIssueSpec, getToolsFacts, processTestRequest} = require('./util');
|
|
11
12
|
|
|
12
|
-
//
|
|
13
|
+
// SETUP
|
|
14
|
+
|
|
15
|
+
const savedDBDir = process.env.DB_DIR;
|
|
13
16
|
|
|
14
17
|
test('getRuleEngineFacts returns the name and sponsor of a known rule engine', () => {
|
|
15
18
|
const facts = getRuleEngineFacts('axe');
|
|
@@ -40,3 +43,66 @@ test('getRuleEnginesFacts returns an empty array for an empty set', () => {
|
|
|
40
43
|
const facts = getRuleEnginesFacts(new Set());
|
|
41
44
|
assert.deepEqual(facts, []);
|
|
42
45
|
});
|
|
46
|
+
|
|
47
|
+
test('getIssueSpec returns a specification for a known non-ignorable issue', () => {
|
|
48
|
+
const spec = getIssueSpec('linkNoText');
|
|
49
|
+
assert.ok(spec);
|
|
50
|
+
assert.ok(spec.summary);
|
|
51
|
+
assert.ok(spec.wcag);
|
|
52
|
+
assert.ok([1, 2, 3, 4].includes(spec.weight));
|
|
53
|
+
assert.ok(spec.why);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('getIssueSpec returns null for an unknown issue ID', () => {
|
|
57
|
+
const spec = getIssueSpec('nonexistentIssue');
|
|
58
|
+
assert.equal(spec, null);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('getIssueSpec returns null for the ignorable issue ID', () => {
|
|
62
|
+
const spec = getIssueSpec('ignorable');
|
|
63
|
+
assert.equal(spec, null);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('getReportBasics returns an error for a nonexistent report', async () => {
|
|
67
|
+
process.env.DB_DIR = path.join(__dirname, '..', 'test', 'fixtures', 'db');
|
|
68
|
+
const basics = await getReportBasics('999999T9999', 'xyz');
|
|
69
|
+
assert.ok(basics.error);
|
|
70
|
+
if (savedDBDir !== undefined) {
|
|
71
|
+
process.env.DB_DIR = savedDBDir;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
delete process.env.DB_DIR;
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test('processTestRequest returns an error for a duplicate recommendation', async () => {
|
|
79
|
+
process.env.DB_DIR = path.join(__dirname, '..', 'test', 'fixtures', 'db');
|
|
80
|
+
// Submit the same request twice; the second should be a duplicate.
|
|
81
|
+
await processTestRequest('test', 'Dup Page', 'https://example.com/dup', 'A reason that is long enough.');
|
|
82
|
+
const result = await processTestRequest('test', 'Dup Page', 'https://example.com/dup', 'A reason that is long enough.');
|
|
83
|
+
assert.equal(result.status, 'error');
|
|
84
|
+
assert.equal(result.message, 'Duplicate request');
|
|
85
|
+
if (savedDBDir !== undefined) {
|
|
86
|
+
process.env.DB_DIR = savedDBDir;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
delete process.env.DB_DIR;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('getResponseMetadata returns an identifier and date string', () => {
|
|
94
|
+
const meta = getResponseMetadata();
|
|
95
|
+
assert.ok(meta.identifier);
|
|
96
|
+
assert.ok(meta.identifier.includes('-'));
|
|
97
|
+
assert.ok(meta['date and time']);
|
|
98
|
+
assert.equal(typeof meta['date and time'], 'string');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('getToolsFacts returns facts about the Kilotest tool collection', () => {
|
|
102
|
+
const facts = getToolsFacts();
|
|
103
|
+
assert.equal(facts.name, 'Kilotest');
|
|
104
|
+
assert.ok(facts.description);
|
|
105
|
+
assert.ok(facts.description['what Kilotest does']);
|
|
106
|
+
assert.ok(facts.description['how to retrieve findings']);
|
|
107
|
+
assert.ok(facts.description['how to generate more findings']);
|
|
108
|
+
});
|
package/apiIntegration.test.js
CHANGED
|
@@ -23,7 +23,7 @@ let server;
|
|
|
23
23
|
before(async () => {
|
|
24
24
|
process.env.DB_DIR = path.join(__dirname, 'test', 'fixtures', 'db');
|
|
25
25
|
server = http.createServer(requestHandler);
|
|
26
|
-
await new Promise(resolve => server.listen(port, resolve));
|
|
26
|
+
await new Promise(resolve => server.listen(port, () => resolve()));
|
|
27
27
|
});
|
|
28
28
|
|
|
29
29
|
after(async () => {
|
|
@@ -33,7 +33,17 @@ after(async () => {
|
|
|
33
33
|
else {
|
|
34
34
|
delete process.env.DB_DIR;
|
|
35
35
|
}
|
|
36
|
-
|
|
36
|
+
server.closeAllConnections?.();
|
|
37
|
+
await new Promise(resolve => {
|
|
38
|
+
const timer = setTimeout(() => {
|
|
39
|
+
server.closeAllConnections?.();
|
|
40
|
+
resolve();
|
|
41
|
+
}, 1000);
|
|
42
|
+
server.close(() => {
|
|
43
|
+
clearTimeout(timer);
|
|
44
|
+
resolve();
|
|
45
|
+
});
|
|
46
|
+
});
|
|
37
47
|
});
|
|
38
48
|
|
|
39
49
|
// HELPERS
|
|
@@ -71,7 +81,7 @@ test('GET /api/listReports returns valid JSON through the HTTP routing layer', a
|
|
|
71
81
|
const {body} = await request('GET', '/api/listReports');
|
|
72
82
|
assert.equal(body['tool name'], 'listReports');
|
|
73
83
|
const reports = body['response content']['basics about all available reports'];
|
|
74
|
-
assert.equal(reports.length,
|
|
84
|
+
assert.equal(reports.length, 8);
|
|
75
85
|
});
|
|
76
86
|
|
|
77
87
|
test('POST /api/requestFeature parses a JSON body and returns a JSON response through the HTTP routing layer', async () => {
|