@jrpool/kilotest 46.0.8 → 46.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +4 -1
- package/.github/workflows/{smoke-test.yml → ci.yml} +8 -2
- package/README.md +14 -8
- package/alerts.test.js +137 -0
- package/api/listDiagnoses.test.js +44 -0
- package/api/listIssues.js +1 -1
- package/api/listIssues.test.js +10 -0
- package/api/listReports.test.js +6 -2
- package/api/listViolators.test.js +35 -0
- package/api/requestTest.test.js +7 -0
- package/api/routes.test.js +63 -0
- package/api/util.js +9 -11
- package/api/util.test.js +68 -2
- package/apiIntegration.test.js +13 -3
- package/docs/AI-TOOLS.md +41 -158
- package/docs/{Smoke Test.json → CI.json} +4 -3
- package/index.html +3 -3
- package/index.js +64 -77
- package/index.test.js +1152 -0
- package/indexBalances.test.js +188 -0
- package/indexFallback.test.js +33 -0
- package/indexServe.test.js +59 -0
- package/indexWorkers.test.js +79 -0
- package/lhm.plugin.json +3218 -0
- package/llms-full.txt +6 -7
- package/llms.txt +1 -1
- package/mcp.js +2 -2
- package/mcp.test.js +281 -0
- package/openapi.yaml +1 -1
- package/package.json +10 -5
- package/routing.test.js +13 -3
- package/scripts/generate-openapi.js +1 -1
- package/test/fixtures/buildFixtures.js +83 -0
- package/test/fixtures/db/reports/260101T0008-mul.json +137 -0
- package/test/fixtures/db/reports/260101T0009-brd.json +147 -0
- package/test/setup.cjs +14 -0
- package/util.js +16 -6
- package/util.test.js +886 -3
- package/web/ai0BalanceForm/ai0BalanceForm.test.js +88 -0
- package/web/expungeReportsForm/expungeReportsForm.test.js +114 -0
- package/web/hideReportForm/hideReportForm.test.js +82 -0
- package/web/listDiagnoses/listDiagnoses.test.js +160 -0
- package/web/listIssues/listIssues.test.js +109 -0
- package/web/listReports/listReports.test.js +139 -3
- package/web/listRules/listRules.test.js +103 -0
- package/web/listTopIssues/listTopIssues.test.js +126 -0
- package/web/listViolators/index.js +1 -2
- package/web/listViolators/listViolators.test.js +256 -0
- package/web/pruneReportsForm/pruneReportsForm.test.js +86 -0
- package/web/reannotate/reannotate.test.js +144 -0
- package/web/reannotateForm/reannotateForm.test.js +134 -0
- package/web/renewWCAG/renewWCAG.test.js +124 -0
- package/web/requestRetest/requestRetest.test.js +119 -0
- package/web/requestRetestForm/requestRetestForm.test.js +62 -0
- package/web/requestTest/requestTest.test.js +115 -0
- package/web/requestTestForm/requestTestForm.test.js +40 -0
- package/web/rewindReportsForm/rewindReportsForm.test.js +86 -0
- package/web/tutorial/index.html +3 -3
- package/web/tutorial/tutorial.test.js +97 -0
- package/web/unhideReportForm/unhideReportForm.test.js +135 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/*
|
|
2
|
+
indexBalances.test.js
|
|
3
|
+
Tests for checkBalancesForAlerts in index.js, which requires balance-related
|
|
4
|
+
env vars to be set before index.js is loaded.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// ENVIRONMENT (must be set before requiring index.js)
|
|
8
|
+
|
|
9
|
+
const path = require('node:path');
|
|
10
|
+
const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
|
|
11
|
+
|
|
12
|
+
process.env.DB_DIR = fixtureDBDir;
|
|
13
|
+
process.env.AUTH_CODE = 'test-auth-code';
|
|
14
|
+
process.env.TESTARO_WORKERS = JSON.stringify({
|
|
15
|
+
worker1: {secret: 'secret1', name: 'Worker One'}
|
|
16
|
+
});
|
|
17
|
+
process.env.WAVE_BALANCE_THRESHOLD = '100';
|
|
18
|
+
process.env.AI_SERVICE0_BALANCE_THRESHOLD = '50';
|
|
19
|
+
process.env.AI_MODEL0_INPUT_PRICE = '0.001';
|
|
20
|
+
process.env.AI_MODEL0_OUTPUT_PRICE = '0.002';
|
|
21
|
+
|
|
22
|
+
const {test, before, after} = require('node:test');
|
|
23
|
+
const assert = require('node:assert/strict');
|
|
24
|
+
const http = require('node:http');
|
|
25
|
+
const fs = require('node:fs/promises');
|
|
26
|
+
const {requestHandler} = require('./index');
|
|
27
|
+
|
|
28
|
+
// CONSTANTS
|
|
29
|
+
|
|
30
|
+
const port = 3994;
|
|
31
|
+
const balancePath = path.join(__dirname, 'ai0Balance.json');
|
|
32
|
+
const jobID = '990101T0001-bal';
|
|
33
|
+
const reportPath = path.join(fixtureDBDir, 'reports', `${jobID}.json`);
|
|
34
|
+
const claimedDir = path.join(fixtureDBDir, 'jobs', 'claimed');
|
|
35
|
+
const jobPath = path.join(claimedDir, `${jobID}.json`);
|
|
36
|
+
|
|
37
|
+
// SETUP AND TEARDOWN
|
|
38
|
+
|
|
39
|
+
let server;
|
|
40
|
+
let originalBalance;
|
|
41
|
+
|
|
42
|
+
before(async () => {
|
|
43
|
+
originalBalance = await fs.readFile(balancePath, 'utf8').catch(() => null);
|
|
44
|
+
server = http.createServer(requestHandler);
|
|
45
|
+
await new Promise(resolve => server.listen(port, () => resolve()));
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
after(async () => {
|
|
49
|
+
server.closeAllConnections?.();
|
|
50
|
+
await new Promise(resolve => {
|
|
51
|
+
const timer = setTimeout(() => {
|
|
52
|
+
server.closeAllConnections?.();
|
|
53
|
+
resolve();
|
|
54
|
+
}, 1000);
|
|
55
|
+
server.close(() => {
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
resolve();
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
if (originalBalance !== null) {
|
|
61
|
+
await fs.writeFile(balancePath, originalBalance);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
await fs.unlink(balancePath).catch(() => {});
|
|
65
|
+
}
|
|
66
|
+
// Clean up any report and job files created by tests.
|
|
67
|
+
await fs.unlink(reportPath).catch(() => {});
|
|
68
|
+
await fs.unlink(jobPath).catch(() => {});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// HELPERS
|
|
72
|
+
|
|
73
|
+
const request = (method, requestPath, body = null, headers = {}) => new Promise((resolve, reject) => {
|
|
74
|
+
const options = {method, host: 'localhost', port, path: requestPath, headers: {...headers}};
|
|
75
|
+
let bodyData = '';
|
|
76
|
+
if (body) {
|
|
77
|
+
bodyData = JSON.stringify(body);
|
|
78
|
+
options.headers['content-type'] = 'application/json';
|
|
79
|
+
options.headers['content-length'] = Buffer.byteLength(bodyData);
|
|
80
|
+
}
|
|
81
|
+
const req = http.request(options, response => {
|
|
82
|
+
const chunks = [];
|
|
83
|
+
response.on('data', chunk => chunks.push(chunk));
|
|
84
|
+
response.on('end', () => {
|
|
85
|
+
resolve({statusCode: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString()});
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
req.on('error', reject);
|
|
89
|
+
req.end(bodyData || '');
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
const jsonBody = res => {
|
|
93
|
+
try {
|
|
94
|
+
return JSON.parse(res.body);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const submitReport = async (acts, balanceFileContent) => {
|
|
102
|
+
if (balanceFileContent !== undefined) {
|
|
103
|
+
if (balanceFileContent === null) {
|
|
104
|
+
await fs.unlink(balancePath).catch(() => {});
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
await fs.writeFile(balancePath, balanceFileContent);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
await fs.writeFile(jobPath, JSON.stringify({
|
|
111
|
+
id: jobID,
|
|
112
|
+
target: {what: 'Test', url: 'https://example.com/test'},
|
|
113
|
+
sources: {worker: 'Worker One'}
|
|
114
|
+
}));
|
|
115
|
+
const auth = Buffer.from('worker1:secret1').toString('base64');
|
|
116
|
+
const report = {
|
|
117
|
+
id: jobID,
|
|
118
|
+
target: {what: 'Test', url: 'https://example.com/test'},
|
|
119
|
+
acts,
|
|
120
|
+
jobData: {endTime: '26-01-01T00:00'},
|
|
121
|
+
catalog: {}
|
|
122
|
+
};
|
|
123
|
+
const res = await request('POST', '/worker/report', {report}, {
|
|
124
|
+
authorization: `Basic ${auth}`
|
|
125
|
+
});
|
|
126
|
+
await fs.unlink(jobPath).catch(() => {});
|
|
127
|
+
await fs.unlink(reportPath).catch(() => {});
|
|
128
|
+
return res;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
// TESTS
|
|
132
|
+
|
|
133
|
+
test('worker/report with WAVE balance low triggers a balance alert', async () => {
|
|
134
|
+
const res = await submitReport(
|
|
135
|
+
[
|
|
136
|
+
{type: 'test', which: 'wave', data: {creditsRemaining: 5}},
|
|
137
|
+
{type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
|
|
138
|
+
],
|
|
139
|
+
JSON.stringify({balance: 100})
|
|
140
|
+
);
|
|
141
|
+
assert.equal(res.statusCode, 200);
|
|
142
|
+
const body = jsonBody(res);
|
|
143
|
+
assert.equal(body.status, 'ok');
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test('worker/report with AI service balance low triggers a balance alert', async () => {
|
|
147
|
+
const res = await submitReport(
|
|
148
|
+
[
|
|
149
|
+
{type: 'test', which: 'wave', data: {creditsRemaining: 200}},
|
|
150
|
+
{type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 5000, outputTokens: 3000}}}}}
|
|
151
|
+
],
|
|
152
|
+
JSON.stringify({balance: 10})
|
|
153
|
+
);
|
|
154
|
+
assert.equal(res.statusCode, 200);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('worker/report with no balance file logs an error', async () => {
|
|
158
|
+
const res = await submitReport(
|
|
159
|
+
[
|
|
160
|
+
{type: 'test', which: 'wave', data: {creditsRemaining: 200}},
|
|
161
|
+
{type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
|
|
162
|
+
],
|
|
163
|
+
null
|
|
164
|
+
);
|
|
165
|
+
assert.equal(res.statusCode, 200);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
test('worker/report with invalid balance JSON logs an error', async () => {
|
|
169
|
+
const res = await submitReport(
|
|
170
|
+
[
|
|
171
|
+
{type: 'test', which: 'wave', data: {creditsRemaining: 200}},
|
|
172
|
+
{type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
|
|
173
|
+
],
|
|
174
|
+
'not valid json'
|
|
175
|
+
);
|
|
176
|
+
assert.equal(res.statusCode, 200);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test('worker/report with non-number balance logs an error', async () => {
|
|
180
|
+
const res = await submitReport(
|
|
181
|
+
[
|
|
182
|
+
{type: 'test', which: 'wave', data: {creditsRemaining: 200}},
|
|
183
|
+
{type: 'test', which: 'testaro', data: {ruleData: {allCaps: {aiModelUsage: {inputTokens: 100, outputTokens: 50}}}}}
|
|
184
|
+
],
|
|
185
|
+
JSON.stringify({balance: 'not a number'})
|
|
186
|
+
);
|
|
187
|
+
assert.equal(res.statusCode, 200);
|
|
188
|
+
});
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/*
|
|
2
|
+
indexFallback.test.js
|
|
3
|
+
Tests that index.js loads successfully when PROTOCOL and TESTARO_WORKERS
|
|
4
|
+
are not set, exercising the || fallback branches at module load time.
|
|
5
|
+
This must be a separate file because index.js reads these env vars
|
|
6
|
+
at module load time, and each test file runs in its own process.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
// ENVIRONMENT (must be set before requiring index.js)
|
|
10
|
+
|
|
11
|
+
const path = require('node:path');
|
|
12
|
+
const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
|
|
13
|
+
|
|
14
|
+
process.env.DB_DIR = fixtureDBDir;
|
|
15
|
+
process.env.AUTH_CODE = 'test-auth-code';
|
|
16
|
+
// Set PROTOCOL and TESTARO_WORKERS to empty strings (falsy) so that
|
|
17
|
+
// the || fallbacks in index.js are exercised. dotenv does not override
|
|
18
|
+
// existing env vars, so .env will not replace these.
|
|
19
|
+
process.env.PROTOCOL = '';
|
|
20
|
+
process.env.TESTARO_WORKERS = '';
|
|
21
|
+
|
|
22
|
+
// IMPORTS
|
|
23
|
+
|
|
24
|
+
const {test} = require('node:test');
|
|
25
|
+
const assert = require('node:assert/strict');
|
|
26
|
+
const {requestHandler} = require('./index');
|
|
27
|
+
|
|
28
|
+
// TESTS
|
|
29
|
+
|
|
30
|
+
test('index.js loads with empty PROTOCOL and TESTARO_WORKERS using fallbacks', () => {
|
|
31
|
+
// If index.js loaded successfully, requestHandler is a function.
|
|
32
|
+
assert.equal(typeof requestHandler, 'function');
|
|
33
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/*
|
|
2
|
+
indexServe.test.js
|
|
3
|
+
Tests for the serve function in index.js, which starts the HTTP or HTTPS
|
|
4
|
+
server. The serve function is exported for testing.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// ENVIRONMENT (must be set before requiring index.js)
|
|
8
|
+
|
|
9
|
+
const path = require('node:path');
|
|
10
|
+
const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
|
|
11
|
+
|
|
12
|
+
process.env.DB_DIR = fixtureDBDir;
|
|
13
|
+
process.env.AUTH_CODE = 'test-auth-code';
|
|
14
|
+
process.env.TESTARO_WORKERS = '{}';
|
|
15
|
+
|
|
16
|
+
const {test} = require('node:test');
|
|
17
|
+
const assert = require('node:assert/strict');
|
|
18
|
+
const http = require('node:http');
|
|
19
|
+
const fs = require('node:fs/promises');
|
|
20
|
+
const {serve} = require('./index');
|
|
21
|
+
|
|
22
|
+
// HELPER
|
|
23
|
+
|
|
24
|
+
const closeServer = server => new Promise(resolve => {
|
|
25
|
+
if (!server) {
|
|
26
|
+
resolve();
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
// Close all idle connections, then close the server.
|
|
30
|
+
server.closeAllConnections?.();
|
|
31
|
+
const timer = setTimeout(() => {
|
|
32
|
+
server.closeAllConnections?.();
|
|
33
|
+
resolve();
|
|
34
|
+
}, 500);
|
|
35
|
+
server.close(() => {
|
|
36
|
+
clearTimeout(timer);
|
|
37
|
+
resolve();
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// TESTS
|
|
42
|
+
|
|
43
|
+
test('serve creates missing directories and returns an HTTP server', async () => {
|
|
44
|
+
// Remove the queue directory to verify serve recreates it.
|
|
45
|
+
const queueDir = path.join(fixtureDBDir, 'jobs', 'queue');
|
|
46
|
+
await fs.rm(queueDir, {recursive: true}).catch(() => {});
|
|
47
|
+
process.env.PORT = '3987';
|
|
48
|
+
const server = await serve(http, {});
|
|
49
|
+
try {
|
|
50
|
+
assert.ok(server);
|
|
51
|
+
assert.equal(typeof server.listen, 'function');
|
|
52
|
+
// Verify the queue directory was recreated.
|
|
53
|
+
const exists = await fs.access(queueDir).then(() => true).catch(() => false);
|
|
54
|
+
assert.ok(exists, 'serve should create the queue directory');
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
await closeServer(server);
|
|
58
|
+
}
|
|
59
|
+
});
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/*
|
|
2
|
+
indexWorkers.test.js
|
|
3
|
+
Tests for index.js when TESTARO_WORKERS is invalid JSON.
|
|
4
|
+
This must be a separate file because index.js reads TESTARO_WORKERS
|
|
5
|
+
at module load time.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// ENVIRONMENT (must be set before requiring index.js)
|
|
9
|
+
|
|
10
|
+
const path = require('node:path');
|
|
11
|
+
const fixtureDBDir = path.join(__dirname, 'test', 'fixtures', 'db');
|
|
12
|
+
|
|
13
|
+
process.env.DB_DIR = fixtureDBDir;
|
|
14
|
+
process.env.AUTH_CODE = 'test-auth-code';
|
|
15
|
+
process.env.TESTARO_WORKERS = 'not valid json';
|
|
16
|
+
|
|
17
|
+
// IMPORTS
|
|
18
|
+
|
|
19
|
+
const {test} = require('node:test');
|
|
20
|
+
const assert = require('node:assert/strict');
|
|
21
|
+
const http = require('node:http');
|
|
22
|
+
const {requestHandler} = require('./index');
|
|
23
|
+
|
|
24
|
+
// CONSTANTS
|
|
25
|
+
|
|
26
|
+
const port = 3993;
|
|
27
|
+
|
|
28
|
+
// HELPERS
|
|
29
|
+
|
|
30
|
+
const request = (method, requestPath, body = null, headers = {}) => new Promise((resolve, reject) => {
|
|
31
|
+
const options = {method, host: 'localhost', port, path: requestPath, headers: {...headers}};
|
|
32
|
+
let bodyData = '';
|
|
33
|
+
if (body) {
|
|
34
|
+
bodyData = JSON.stringify(body);
|
|
35
|
+
options.headers['content-type'] = 'application/json';
|
|
36
|
+
options.headers['content-length'] = Buffer.byteLength(bodyData);
|
|
37
|
+
}
|
|
38
|
+
const req = http.request(options, response => {
|
|
39
|
+
const chunks = [];
|
|
40
|
+
response.on('data', chunk => chunks.push(chunk));
|
|
41
|
+
response.on('end', () => {
|
|
42
|
+
resolve({statusCode: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString()});
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
req.on('error', reject);
|
|
46
|
+
req.end(bodyData || '');
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// TESTS
|
|
50
|
+
|
|
51
|
+
test('index.js loads with invalid TESTARO_WORKERS and treats workers as empty', () => {
|
|
52
|
+
// If index.js loaded successfully, requestHandler is a function.
|
|
53
|
+
assert.equal(typeof requestHandler, 'function');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('POST /worker/job with invalid TESTARO_WORKERS returns 401 for any credentials', async () => {
|
|
57
|
+
const server = http.createServer(requestHandler);
|
|
58
|
+
await new Promise(resolve => server.listen(port, () => resolve()));
|
|
59
|
+
try {
|
|
60
|
+
const auth = Buffer.from('worker1:secret1').toString('base64');
|
|
61
|
+
const res = await request('POST', '/worker/job', {}, {
|
|
62
|
+
authorization: `Basic ${auth}`
|
|
63
|
+
});
|
|
64
|
+
assert.equal(res.statusCode, 401);
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
server.closeAllConnections?.();
|
|
68
|
+
await new Promise(resolve => {
|
|
69
|
+
const timer = setTimeout(() => {
|
|
70
|
+
server.closeAllConnections?.();
|
|
71
|
+
resolve();
|
|
72
|
+
}, 1000);
|
|
73
|
+
server.close(() => {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
resolve();
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
});
|