@jrpool/kilotest 24.1.1 → 25.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/env.example +16 -5
- package/index.js +63 -17
- package/package.json +1 -1
- package/reportIssues/api.js +108 -233
- package/reportIssues/index.js +3 -4
- package/reportIssues/util.js +1 -7
- package/researchAgent.js +95 -0
- package/testOrder/index.js +2 -2
- package/util.js +9 -3
package/env.example
CHANGED
|
@@ -1,12 +1,23 @@
|
|
|
1
|
-
#
|
|
2
|
-
ENVIRONMENT=
|
|
3
|
-
# PROTOCOL
|
|
1
|
+
# ENVIRONMENT=test for a dev/test host, =production for a deployed host.
|
|
2
|
+
ENVIRONMENT=__placeholder__
|
|
3
|
+
# PROTOCOL=http for a local dev/test host or reverse-proxied server, otherwise =https.
|
|
4
4
|
PROTOCOL=__placeholder__
|
|
5
|
-
# AUTH_CODE must be 3 characters long.
|
|
5
|
+
# AUTH_CODE for manager operations; must be 3 characters long.
|
|
6
6
|
AUTH_CODE=__placeholder__
|
|
7
|
+
# Name of Testaro agent.
|
|
7
8
|
TESTARO_AGENT=testaro-agent
|
|
8
|
-
#
|
|
9
|
+
# Testaro agent password (same as NETWATCH_AUTH in the Testaro .env.
|
|
9
10
|
TESTARO_AGENT_PW=__placeholder__
|
|
11
|
+
# Name of research agent (AI agent calling Kilotest as a tool).
|
|
12
|
+
RESEARCH_AGENT=research-agent
|
|
13
|
+
# Research agent password.
|
|
14
|
+
RESEARCH_AGENT_PW=__placeholder__
|
|
15
|
+
# URL of this Kilotest host.
|
|
16
|
+
THIS_KILOTEST_HOST=__placeholder__
|
|
17
|
+
# URL of a dev/test Kilotest host.
|
|
18
|
+
LOCAL_KILOTEST_HOST=http://localhost:3000
|
|
19
|
+
# URL of the deployed Kilotest host.
|
|
20
|
+
DEPLOYED_KILOTEST_HOST=https://kilotest.com
|
|
10
21
|
# Alert email configuration:
|
|
11
22
|
MANAGER_EMAIL=__placeholder__
|
|
12
23
|
ALERT_API_HOST=api.resend.com
|
package/index.js
CHANGED
|
@@ -67,6 +67,8 @@ const claimedPath = path.join(jobsPath, 'claimed');
|
|
|
67
67
|
const failedPath = path.join(jobsPath, 'failed');
|
|
68
68
|
const testaroAgent = process.env.TESTARO_AGENT;
|
|
69
69
|
const testaroAgentPW = process.env.TESTARO_AGENT_PW;
|
|
70
|
+
const researchAgent = process.env.RESEARCH_AGENT;
|
|
71
|
+
const researchAgentPW = process.env.RESEARCH_AGENT_PW;
|
|
70
72
|
// Values that may require alerts.
|
|
71
73
|
const balancePath = path.join(__dirname, 'aiService0Balance.json');
|
|
72
74
|
const WAVE_THRESHOLD = Number(process.env.WAVE_BALANCE_THRESHOLD);
|
|
@@ -76,21 +78,29 @@ const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
|
|
|
76
78
|
|
|
77
79
|
// FUNCTIONS
|
|
78
80
|
|
|
79
|
-
// Serves an error message.
|
|
81
|
+
// Serves or sends an error message.
|
|
80
82
|
const serveError = async (error, response, isHumanUser = true) => {
|
|
81
|
-
console.log(error.message);
|
|
83
|
+
console.log(error.message || 'ERROR');
|
|
82
84
|
if (! response.writableEnded) {
|
|
83
85
|
response.statusCode = 400;
|
|
86
|
+
// If the request is from a human user:
|
|
84
87
|
if (isHumanUser) {
|
|
88
|
+
// Serve an HTML page containing the message property of the error.
|
|
85
89
|
response.setHeader('content-type', 'text/html; charset=utf-8');
|
|
86
90
|
const errorTemplate = await fs.readFile('error.html', 'utf8');
|
|
87
|
-
const errorPage = errorTemplate.replace(/__error__/, error.message);
|
|
91
|
+
const errorPage = errorTemplate.replace(/__error__/, error.message || 'ERROR');
|
|
88
92
|
response.end(errorPage);
|
|
89
|
-
}
|
|
93
|
+
}
|
|
94
|
+
// Otherwise, i.e. if it is from an agent:
|
|
95
|
+
else {
|
|
96
|
+
// Send a JSON response containing the entire error.
|
|
90
97
|
response.setHeader('content-type', 'application/json; charset=utf-8');
|
|
91
|
-
response.end(JSON.stringify({error
|
|
98
|
+
response.end(JSON.stringify({error}));
|
|
92
99
|
}
|
|
93
100
|
}
|
|
101
|
+
else {
|
|
102
|
+
console.log('Cannot send error response because the response has ended.');
|
|
103
|
+
}
|
|
94
104
|
};
|
|
95
105
|
// Checks a report for balances nearing exhaustion.
|
|
96
106
|
const checkBalancesForAlerts = async report => {
|
|
@@ -107,7 +117,6 @@ const checkBalancesForAlerts = async report => {
|
|
|
107
117
|
`Only ${creditsRemaining} WAVE credits remain (3 used per job)`
|
|
108
118
|
);
|
|
109
119
|
}
|
|
110
|
-
// AI service 0.
|
|
111
120
|
const testaroAct = report.acts.find(act => act.type === 'test' && act.which === 'testaro');
|
|
112
121
|
// Get the AI model token usage for the testaro allCaps test.
|
|
113
122
|
const usage = testaroAct?.data?.ruleData?.allCaps?.aiModelUsage;
|
|
@@ -231,7 +240,7 @@ const requestHandler = async (request, response) => {
|
|
|
231
240
|
// Otherwise, i.e. if the request is syntactically invalid:
|
|
232
241
|
else {
|
|
233
242
|
// Report the error.
|
|
234
|
-
await serveError({message: 'Invalid report request'}, response, true);
|
|
243
|
+
await serveError({message: 'ERROR: Invalid report request'}, response, true);
|
|
235
244
|
}
|
|
236
245
|
}
|
|
237
246
|
// Otherwise, if it is for an HTML page other than the home page:
|
|
@@ -258,7 +267,7 @@ const requestHandler = async (request, response) => {
|
|
|
258
267
|
// Otherwise, i.e. if the answer cannot be generated:
|
|
259
268
|
else {
|
|
260
269
|
// Report the error.
|
|
261
|
-
await serveError({message: 'Invalid request'}, response, true);
|
|
270
|
+
await serveError({message: 'ERROR: Invalid request'}, response, true);
|
|
262
271
|
}
|
|
263
272
|
}
|
|
264
273
|
// Otherwise, if it is for a tutorial image:
|
|
@@ -274,7 +283,7 @@ const requestHandler = async (request, response) => {
|
|
|
274
283
|
response.end(img);
|
|
275
284
|
}
|
|
276
285
|
catch (_) {
|
|
277
|
-
await serveError({message: 'Image not found'}, response, true);
|
|
286
|
+
await serveError({message: 'ERROR: Image not found'}, response, true);
|
|
278
287
|
}
|
|
279
288
|
}
|
|
280
289
|
// Otherwise, if it is for the application icon:
|
|
@@ -340,7 +349,7 @@ const requestHandler = async (request, response) => {
|
|
|
340
349
|
// Otherwise, i.e. if the request is invalid:
|
|
341
350
|
else {
|
|
342
351
|
// Report the error.
|
|
343
|
-
await serveError({message: 'Invalid retest recommendation'}, response, true);
|
|
352
|
+
await serveError({message: 'ERROR: Invalid retest recommendation'}, response, true);
|
|
344
353
|
}
|
|
345
354
|
}
|
|
346
355
|
// Otherwise, if it is a test recommendation:
|
|
@@ -368,7 +377,7 @@ const requestHandler = async (request, response) => {
|
|
|
368
377
|
// Otherwise, i.e. if the request is invalid:
|
|
369
378
|
else {
|
|
370
379
|
// Report the error.
|
|
371
|
-
await serveError({message: 'Invalid test recommendation'}, response, true);
|
|
380
|
+
await serveError({message: 'ERROR: Invalid test recommendation'}, response, true);
|
|
372
381
|
}
|
|
373
382
|
}
|
|
374
383
|
// Otherwise, if it is an action on a test or retest recommendation:
|
|
@@ -416,7 +425,7 @@ const requestHandler = async (request, response) => {
|
|
|
416
425
|
// Otherwise, i.e. if the request is invalid:
|
|
417
426
|
else {
|
|
418
427
|
// Report the error.
|
|
419
|
-
await serveError({message: 'Invalid test order'}, response, true);
|
|
428
|
+
await serveError({message: 'ERROR: Invalid test order'}, response, true);
|
|
420
429
|
}
|
|
421
430
|
}
|
|
422
431
|
// Otherwise, if it is a reannotation order:
|
|
@@ -459,10 +468,11 @@ const requestHandler = async (request, response) => {
|
|
|
459
468
|
await serveError({message: answerData.error}, response, true);
|
|
460
469
|
}
|
|
461
470
|
}
|
|
462
|
-
// Otherwise, if it is a request from
|
|
471
|
+
// Otherwise, if it is a request from an agent:
|
|
463
472
|
else if (pageName === 'api') {
|
|
464
|
-
|
|
465
|
-
|
|
473
|
+
// Get the agent ID, the service, and any service specifications from the path.
|
|
474
|
+
const [agentID, service, ... specs] = pageArgs.split('/');
|
|
475
|
+
// If the agent is the authorized Testaro instance and it is authenticated:
|
|
466
476
|
if (agentID === testaroAgent && postData.agentPW === testaroAgentPW) {
|
|
467
477
|
// If the service is job assignment:
|
|
468
478
|
if (service === 'job') {
|
|
@@ -570,9 +580,42 @@ const requestHandler = async (request, response) => {
|
|
|
570
580
|
);
|
|
571
581
|
}
|
|
572
582
|
}
|
|
573
|
-
// Otherwise,
|
|
583
|
+
// Otherwise, if the agent is the authorized research agent and it is authenticated:
|
|
584
|
+
else if (agentID === researchAgent && postData.agentPW === researchAgentPW) {
|
|
585
|
+
// If the service is provision of facts about issues in a report:
|
|
586
|
+
if (service === 'reportIssues') {
|
|
587
|
+
// Get the report identifiers from the path.
|
|
588
|
+
const [timeStamp, jobID] = specs;
|
|
589
|
+
const args = [agentID, timeStamp, jobID];
|
|
590
|
+
const reportSpecsBad = await isHidden(timeStamp, jobID);
|
|
591
|
+
// If the report is nonexistent or hidden:
|
|
592
|
+
if (reportSpecsBad) {
|
|
593
|
+
// Report this.
|
|
594
|
+
await serveError(
|
|
595
|
+
{message: reportSpecsBad === true ? 'Report nonexistent or hidden' : reportSpecsBad},
|
|
596
|
+
response,
|
|
597
|
+
false
|
|
598
|
+
);
|
|
599
|
+
}
|
|
600
|
+
// Otherwise, i.e. if the report is available:
|
|
601
|
+
else {
|
|
602
|
+
// Get the response (potentially error) data.
|
|
603
|
+
const responseData = await require(path.join(__dirname, 'reportIssues', 'api'))
|
|
604
|
+
.response(args);
|
|
605
|
+
// Send them.
|
|
606
|
+
response.end(JSON.stringify(responseData));
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
// Otherwise, i.e. if the service is invalid:
|
|
610
|
+
else {
|
|
611
|
+
// Report this.
|
|
612
|
+
await serveError({message: 'ERROR: Invalid service request'}, response, false);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
// Otherwise, i.e. if the agent is not authorized or not authenticated:
|
|
574
616
|
else {
|
|
575
|
-
|
|
617
|
+
// Report this.
|
|
618
|
+
await serveError({message: 'ERROR: Invalid agent'}, response, false);
|
|
576
619
|
}
|
|
577
620
|
}
|
|
578
621
|
// Otherwise, if it is a tutorial comment:
|
|
@@ -617,6 +660,9 @@ const serve = async (protocolModule, options) => {
|
|
|
617
660
|
console.log(`Kilotest server listening at ${protocol}://localhost:${port}.`);
|
|
618
661
|
});
|
|
619
662
|
};
|
|
663
|
+
|
|
664
|
+
// EXECUTION
|
|
665
|
+
|
|
620
666
|
if (protocol === 'http') {
|
|
621
667
|
console.log('Starting HTTP server');
|
|
622
668
|
serve(http, {});
|
package/package.json
CHANGED
package/reportIssues/api.js
CHANGED
|
@@ -1,231 +1,57 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
api.js
|
|
3
|
+
Responds to the report-issues request.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// IMPORTS
|
|
7
7
|
|
|
8
|
+
const {getData, getToolData} = require('./util');
|
|
8
9
|
const {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
getWCAGLink,
|
|
13
|
-
getWeightName,
|
|
14
|
-
htmlSafe,
|
|
10
|
+
getDateTime,
|
|
11
|
+
getNowStamp,
|
|
12
|
+
getRandomString,
|
|
15
13
|
isHidden,
|
|
16
|
-
|
|
17
|
-
objectSort,
|
|
14
|
+
researchAgents,
|
|
18
15
|
tools
|
|
19
16
|
} = require('../util');
|
|
20
|
-
const {issues} = require('testilo/procs/score/tic');
|
|
21
|
-
const fs = require('fs/promises');
|
|
22
|
-
const path = require('path');
|
|
23
17
|
|
|
24
18
|
// FUNCTIONS
|
|
25
19
|
|
|
26
|
-
// Gets
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
reportersString: '',
|
|
36
|
-
violators: new Set(),
|
|
37
|
-
violatorCount: 0,
|
|
38
|
-
preventions: {},
|
|
39
|
-
issuesObject: {},
|
|
40
|
-
issueCount: 0,
|
|
41
|
-
issues: []
|
|
20
|
+
// Gets facts about tools.
|
|
21
|
+
const getToolFacts = toolIDs => {
|
|
22
|
+
const crypticData = getToolData(toolIDs);
|
|
23
|
+
return crypticData.map(tool => {
|
|
24
|
+
const {toolID, toolName, toolMaker} = tool;
|
|
25
|
+
return {
|
|
26
|
+
identifier: toolID,
|
|
27
|
+
name: toolName,
|
|
28
|
+
sponsor: toolMaker
|
|
42
29
|
};
|
|
43
|
-
const {issuesObject, reporters, violators} = issuesData;
|
|
44
|
-
// For each act in it:
|
|
45
|
-
report.acts.forEach(act => {
|
|
46
|
-
// If it is a test act:
|
|
47
|
-
if (act.type === 'test') {
|
|
48
|
-
const {result, which} = act;
|
|
49
|
-
const instances = result?.standardResult?.instances ?? [];
|
|
50
|
-
// For each of its standard instances:
|
|
51
|
-
instances.forEach(instance => {
|
|
52
|
-
const {issueID} = instance;
|
|
53
|
-
// If the instance has a non-ignorable classified issue:
|
|
54
|
-
if (issueID && issues[issueID] && issueID !== 'ignorable') {
|
|
55
|
-
// Ensure that the issues data include data on the issue.
|
|
56
|
-
issuesObject[issueID] ??= {
|
|
57
|
-
issueID,
|
|
58
|
-
weight: issues[issueID].weight ?? 0,
|
|
59
|
-
reporters: new Set(),
|
|
60
|
-
reporterCount: 0,
|
|
61
|
-
reportersString: '',
|
|
62
|
-
violators: new Set(),
|
|
63
|
-
violatorCount: 0
|
|
64
|
-
};
|
|
65
|
-
// Ensure that the tool is in the issues data.
|
|
66
|
-
reporters.add(which);
|
|
67
|
-
// Ensure that it is in the issue data.
|
|
68
|
-
issuesObject[issueID].reporters.add(which);
|
|
69
|
-
const {catalogIndex} = instance;
|
|
70
|
-
// If the instance has a catalog index:
|
|
71
|
-
if (catalogIndex) {
|
|
72
|
-
// Ensure that the violator is in the issues data.
|
|
73
|
-
violators.add(catalogIndex);
|
|
74
|
-
// Ensure that it is in the issue data.
|
|
75
|
-
issuesObject[issueID].violators.add(catalogIndex);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
});
|
|
81
|
-
// Populate the unpopulated subproperties of the issues data.
|
|
82
|
-
issuesData.reporterCount = issuesData.reporters.size;
|
|
83
|
-
issuesData.reportersString = getToolNamesString(issuesData.reporters);
|
|
84
|
-
issuesData.violatorCount = issuesData.violators.size;
|
|
85
|
-
issuesData.preventions = report.jobData.preventions;
|
|
86
|
-
issuesData.issueCount = Object.keys(issuesData.issuesObject).length;
|
|
87
|
-
issuesData.issues = Object.values(issuesData.issuesObject);
|
|
88
|
-
// For each issue in the issues data:
|
|
89
|
-
issuesData.issues.forEach(issue => {
|
|
90
|
-
// Populate its unpopulated properties.
|
|
91
|
-
issue.reporterCount = issue.reporters.size;
|
|
92
|
-
issue.reportersString = getToolNamesString(issue.reporters);
|
|
93
|
-
issue.violatorCount = issue.violators.size;
|
|
94
|
-
});
|
|
95
|
-
// Sort the issues alphabetically by reporters string.
|
|
96
|
-
objectSort(issuesData.issues, 'reportersString', 'alpha');
|
|
97
|
-
// Sort the issues again in descending reporter-count order, making this the primary order.
|
|
98
|
-
objectSort(issuesData.issues, 'reporterCount', 'numericDown');
|
|
99
|
-
// Return the issues data.
|
|
100
|
-
return issuesData;
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
// Adds parameters to a query for the answer page.
|
|
104
|
-
const populateQuery = async (timeStamp, jobID, query) => {
|
|
105
|
-
// Get fact descriptions for the report.
|
|
106
|
-
const pageDataStrings = await getPageDataStrings(timeStamp, jobID);
|
|
107
|
-
const {what, urlLink, testInfo} = pageDataStrings;
|
|
108
|
-
const issuesData = await getIssuesData(timeStamp, jobID);
|
|
109
|
-
// If this failed:
|
|
110
|
-
if (typeof issuesData === 'string') {
|
|
111
|
-
// Return this.
|
|
112
|
-
return issuesData;
|
|
113
|
-
}
|
|
114
|
-
const {
|
|
115
|
-
issueCount,
|
|
116
|
-
preventions,
|
|
117
|
-
reporterCount,
|
|
118
|
-
reportersString,
|
|
119
|
-
violatorCount
|
|
120
|
-
} = issuesData;
|
|
121
|
-
// Add an issue count description to the query.
|
|
122
|
-
query.issueCount = issueCount === 1 ? '1 issue was' : `${issueCount} issues were`;
|
|
123
|
-
query.reporterCount = reporterCount === 1 ? '1 tool' : `${reporterCount} tools`;
|
|
124
|
-
// Add a reporter count and list to the query.
|
|
125
|
-
query.reporters = reportersString;
|
|
126
|
-
// Add a violator count to the query.
|
|
127
|
-
query.violatorCount = violatorCount === 1 ? '1 violator was' : `${violatorCount} violators were`;
|
|
128
|
-
// Add page data to the query.
|
|
129
|
-
query.target = what;
|
|
130
|
-
query.urlLink = urlLink;
|
|
131
|
-
query.testInfo = testInfo;
|
|
132
|
-
query.timeStamp = timeStamp;
|
|
133
|
-
query.jobID = jobID;
|
|
134
|
-
const preventionStrings = [];
|
|
135
|
-
const margin = ' '.repeat(6);
|
|
136
|
-
Object.keys(preventions).forEach(preventedToolID => {
|
|
137
|
-
const toolName = tools[preventedToolID];
|
|
138
|
-
const toolNameString = `${toolName[0]} (${toolName[1]})`;
|
|
139
|
-
const causeString = htmlSafe(preventions[preventedToolID]);
|
|
140
|
-
const preventionString = `${margin}<li>Page not testable by ${toolNameString}: ${causeString}</li>`;
|
|
141
|
-
preventionStrings.push(preventionString);
|
|
142
30
|
});
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
weightData.push({
|
|
162
|
-
issueID,
|
|
163
|
-
summary: issue.summary,
|
|
164
|
-
why,
|
|
165
|
-
wcag: wcagLink,
|
|
166
|
-
reporterCount,
|
|
167
|
-
reportersString,
|
|
168
|
-
violatorCount
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
});
|
|
172
|
-
const weightName = getWeightName(weight);
|
|
173
|
-
// Add the issue count to the query.
|
|
174
|
-
query[`${weightName}Count`] = weightData.length;
|
|
175
|
-
// If any reported issues have the weight:
|
|
176
|
-
if (weightData.length) {
|
|
177
|
-
// Add the start of a list of the issues with the weight to the lines.
|
|
178
|
-
weightLines.push(`${margin}<ul class="headed">`);
|
|
179
|
-
// For each issue with the weight:
|
|
180
|
-
weightData.forEach(weightIssue => {
|
|
181
|
-
const {issueID, reporterCount, reportersString, violatorCount, wcag, why} = weightIssue;
|
|
182
|
-
// Add the start of a list item to the lines.
|
|
183
|
-
weightLines.push(`${margin} <li>`);
|
|
184
|
-
// Add a heading summarizing the issue to the lines.
|
|
185
|
-
weightLines.push(`${margin} <h5>${weightIssue.summary}</h5>`);
|
|
186
|
-
// Add the start of alist of facts about the issue to the lines.
|
|
187
|
-
weightLines.push(`${margin} <ul class="pseudoTopLevel">`);
|
|
188
|
-
// Add the issue facts to the lines.
|
|
189
|
-
weightLines.push(`${margin} <li>Why it matters: ${why}`);
|
|
190
|
-
weightLines.push(`${margin} <li>Related WCAG standard: ${wcag}`);
|
|
191
|
-
const reporterCountString = reporterCount === 1 ? '1 tool' : `${reporterCount} tools`;
|
|
192
|
-
weightLines.push(
|
|
193
|
-
`${margin} <li>Reported by ${reporterCountString} (${reportersString})</li>`
|
|
194
|
-
);
|
|
195
|
-
const violatorCountString = violatorCount === 1
|
|
196
|
-
? '1 violator was'
|
|
197
|
-
: `${violatorCount} violators were`;
|
|
198
|
-
weightLines.push(`${margin} <li>${violatorCountString} reported</li>`);
|
|
199
|
-
// Add the end of the fact list to the lines.
|
|
200
|
-
weightLines.push(`${margin} </ul>`);
|
|
201
|
-
// Add the start of a link list to the lines.
|
|
202
|
-
weightLines.push(`${margin} <ul class="nav">`);
|
|
203
|
-
const whereQuestionString = 'Where was the issue found?';
|
|
204
|
-
const labelString = `Where was the ${weightIssue.summary} issue found on the ${what} page?`;
|
|
205
|
-
const href = `href="/reportIssue.html/${issueID}/${timeStamp}/${jobID}"`;
|
|
206
|
-
const label = `aria-label="${labelString}"`;
|
|
207
|
-
const whereLink = `<a ${href} ${label}>${whereQuestionString}</a>`;
|
|
208
|
-
// Add a violations link to the lines.
|
|
209
|
-
weightLines.push(`${margin} <li>${whereLink}</li>`);
|
|
210
|
-
// Add the end of the link list to the lines.
|
|
211
|
-
weightLines.push(`${margin} </ul>`);
|
|
212
|
-
// Add the end of the list item to the lines.
|
|
213
|
-
weightLines.push(`${margin} </li>`);
|
|
214
|
-
});
|
|
215
|
-
// Add the end of the list of issues with the weight to the lines.
|
|
216
|
-
weightLines.push(`${margin}</ul>`);
|
|
217
|
-
// Add the lines documenting the issues with the weight to the query.
|
|
218
|
-
query[`${weightName}Details`] = weightLines.join('\n');
|
|
219
|
-
}
|
|
220
|
-
// Otherwise, i.e. if no reported issues have the weight:
|
|
221
|
-
else {
|
|
222
|
-
query[`${weightName}Details`] = `${margin} <p>None</p>`;
|
|
31
|
+
};
|
|
32
|
+
// Gets facts about an issue.
|
|
33
|
+
const getIssueFacts = (thisHost, agentID, timeStamp, jobID, issue) => {
|
|
34
|
+
const {issueID, reporterCount, reporters, summary, violatorCount, wcag, why} = issue;
|
|
35
|
+
const wcagType = wcag.length === 3 ? 'principle' : 'success criterion';
|
|
36
|
+
return {
|
|
37
|
+
identifier: issueID,
|
|
38
|
+
summary,
|
|
39
|
+
[`related WCAG ${wcagType}`]: wcag,
|
|
40
|
+
'impact on a user': why,
|
|
41
|
+
'tools reporting the issue': {
|
|
42
|
+
'number': reporterCount,
|
|
43
|
+
'names': reporters.map(tool => tool.toolName)
|
|
44
|
+
},
|
|
45
|
+
'number of HTML elements reported as exhibiting the issue': violatorCount,
|
|
46
|
+
'URLs for details about the issue on the page': {
|
|
47
|
+
'for you': `${thisHost}/api/${agentID}/reportIssue/${timeStamp}/${jobID}/${issueID}`,
|
|
48
|
+
'for humans': `${thisHost}/reportIssue/${timeStamp}/${jobID}/${issueID}`
|
|
223
49
|
}
|
|
224
|
-
}
|
|
50
|
+
};
|
|
225
51
|
};
|
|
226
|
-
// Returns a
|
|
227
|
-
exports.
|
|
228
|
-
const [timeStamp, jobID] =
|
|
52
|
+
// Returns a response to a target-issues request.
|
|
53
|
+
exports.response = async args => {
|
|
54
|
+
const [agentID, timeStamp, jobID] = args;
|
|
229
55
|
const reportIsHidden = await isHidden(timeStamp, jobID);
|
|
230
56
|
// If the report is not available:
|
|
231
57
|
if (reportIsHidden) {
|
|
@@ -234,26 +60,75 @@ exports.answer = async pageArgs => {
|
|
|
234
60
|
message: 'Report not available'
|
|
235
61
|
};
|
|
236
62
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
63
|
+
// Otherwise, i.e. if the report is available, get data on the target and issues.
|
|
64
|
+
const data = await getData(timeStamp, jobID);
|
|
65
|
+
const {pageData, issuesData} = data;
|
|
66
|
+
const {what, url, daysAgo} = pageData;
|
|
67
|
+
const {issueCount, issues, preventions, reporterCount, reporters, violatorCount} = issuesData;
|
|
68
|
+
const preventedTools = Object.entries(preventions).map(prevention => ({
|
|
69
|
+
name: tools[prevention[0]][0],
|
|
70
|
+
'reason for failure': prevention[1]
|
|
71
|
+
}));
|
|
72
|
+
const thisHost = process.env.THIS_KILOTEST_HOST;
|
|
73
|
+
// Get a response.
|
|
74
|
+
const response = {
|
|
75
|
+
summary: `This document fulfills a request made by an agent to the Kilotest service. The agent requested data from a Kilotest report about the accessibility, usability, and standard-conformity of a web page. Kilotest, with the help of Testaro, Testilo, and an ensemble of ten testing tools, performs tests on web pages, using a combination of rule- and machine-learning-based methods, and produces reports. Kilotest exposes several API endpoints for agents and several web UI URLs for humans to obtain information from Kilotest reports. To learn more about Kilotest and the advangages of testing with an ensemble of tools, visit the deployed instance of Kilotest (${process.env.DEPLOYED_KILOTEST_HOST}), which contains an introduction on its home page and a tutorial.`,
|
|
76
|
+
'tool name': 'Kilotest',
|
|
77
|
+
request: {
|
|
78
|
+
'requesting agent': {
|
|
79
|
+
identifier: agentID,
|
|
80
|
+
name: researchAgents[agentID]
|
|
81
|
+
},
|
|
82
|
+
'type of request': {
|
|
83
|
+
identifier: 'reportIssues',
|
|
84
|
+
description: 'What issues does the specified report describe?'
|
|
85
|
+
},
|
|
86
|
+
'closest ancestor request': {
|
|
87
|
+
description: 'Which web pages are reports available about, and what are the statistics about the issues reported for each page?',
|
|
88
|
+
'URL for you': `${thisHost}/api/${agentID}/targets.html`,
|
|
89
|
+
'URL for humans': `${thisHost}/targets.html`
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
'response metadata': {
|
|
93
|
+
identifier: `${getNowStamp()}-${getRandomString(3)}`,
|
|
94
|
+
'date and time': new Date().toISOString(),
|
|
95
|
+
'URL of the human-oriented equivalent of this response': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
|
|
96
|
+
},
|
|
97
|
+
report: {
|
|
98
|
+
identifier: `${timeStamp}-${jobID}`,
|
|
99
|
+
'creation date': getDateTime(timeStamp),
|
|
100
|
+
'days since the creation date': daysAgo
|
|
101
|
+
},
|
|
102
|
+
'tested web page': {
|
|
103
|
+
description: what,
|
|
104
|
+
URL: url
|
|
105
|
+
},
|
|
106
|
+
'tools that tried to test the page': getToolFacts(Object.keys(tools)),
|
|
107
|
+
'tools that were unable to test the page': preventedTools,
|
|
108
|
+
'tools that reported issues': {
|
|
109
|
+
number: reporterCount,
|
|
110
|
+
names: reporters.map(tool => tool.toolName)
|
|
111
|
+
},
|
|
112
|
+
'number of issues reported': {
|
|
113
|
+
total: issueCount,
|
|
114
|
+
'by priority': {
|
|
115
|
+
'highest priority': issues[4].length,
|
|
116
|
+
'high priority': issues[3].length,
|
|
117
|
+
'low priority': issues[2].length,
|
|
118
|
+
'lowest priority': issues[1].length
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
'number of HTML elements reported as exhibiting issues': violatorCount,
|
|
122
|
+
'issues reported': {
|
|
123
|
+
'highest priority': issues[4]
|
|
124
|
+
.map(issue => getIssueFacts(thisHost, agentID, timeStamp, jobID, issue)),
|
|
125
|
+
'high priority': issues[3]
|
|
126
|
+
.map(issue => getIssueFacts(thisHost, agentID, timeStamp, jobID, issue)),
|
|
127
|
+
'low priority': issues[2]
|
|
128
|
+
.map(issue => getIssueFacts(thisHost, agentID, timeStamp, jobID, issue)),
|
|
129
|
+
'lowest priority': issues[1]
|
|
130
|
+
.map(issue => getIssueFacts(thisHost, agentID, timeStamp, jobID, issue))
|
|
131
|
+
}
|
|
258
132
|
};
|
|
133
|
+
return response;
|
|
259
134
|
};
|
package/reportIssues/index.js
CHANGED
|
@@ -14,7 +14,6 @@ const {
|
|
|
14
14
|
isHidden,
|
|
15
15
|
tools
|
|
16
16
|
} = require('../util');
|
|
17
|
-
const {issues} = require('testilo/procs/score/tic');
|
|
18
17
|
const fs = require('fs/promises');
|
|
19
18
|
const path = require('path');
|
|
20
19
|
|
|
@@ -22,7 +21,7 @@ const path = require('path');
|
|
|
22
21
|
|
|
23
22
|
// Adds parameters to a query for the answer page.
|
|
24
23
|
const populateQuery = async (timeStamp, jobID, query) => {
|
|
25
|
-
// Get data on the
|
|
24
|
+
// Get data on the target and its issues according to the report.
|
|
26
25
|
const data = await getData(timeStamp, jobID);
|
|
27
26
|
const {pageData, issuesData} = data;
|
|
28
27
|
// If the page data are invalid:
|
|
@@ -35,10 +34,10 @@ const populateQuery = async (timeStamp, jobID, query) => {
|
|
|
35
34
|
// Return this.
|
|
36
35
|
return issuesData;
|
|
37
36
|
}
|
|
38
|
-
// Otherwise, get fact descriptions for the
|
|
37
|
+
// Otherwise, get fact descriptions for the target.
|
|
39
38
|
const pageInfo = await getPageDataStrings(timeStamp, jobID, pageData);
|
|
40
39
|
const {what, urlLink, testInfo} = pageInfo;
|
|
41
|
-
// Add
|
|
40
|
+
// Add target data to the query.
|
|
42
41
|
query.target = what;
|
|
43
42
|
query.urlLink = urlLink;
|
|
44
43
|
query.testInfo = testInfo;
|
package/reportIssues/util.js
CHANGED
|
@@ -11,7 +11,7 @@ const issuesClassification = require('testilo/procs/score/tic').issues;
|
|
|
11
11
|
// FUNCTIONS
|
|
12
12
|
|
|
13
13
|
// Converts tool IDs to tool data sorted by tool name.
|
|
14
|
-
const getToolData = toolIDs => objectSort(
|
|
14
|
+
const getToolData = exports.getToolData = toolIDs => objectSort(
|
|
15
15
|
Array.from(toolIDs).map(toolID => {
|
|
16
16
|
const toolData = tools[toolID];
|
|
17
17
|
return {
|
|
@@ -47,12 +47,6 @@ const getIssuesData = async (timeStamp, jobID) => {
|
|
|
47
47
|
reporterCount: 0,
|
|
48
48
|
violatorCount: 0,
|
|
49
49
|
preventions: report.jobData.preventions,
|
|
50
|
-
priorityNames: {
|
|
51
|
-
4: 'highest',
|
|
52
|
-
3: 'high',
|
|
53
|
-
2: 'low',
|
|
54
|
-
1: 'lowest'
|
|
55
|
-
},
|
|
56
50
|
issues: {
|
|
57
51
|
4: [],
|
|
58
52
|
3: [],
|
package/researchAgent.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/*
|
|
2
|
+
© 2026 Jonathan Robert Pool.
|
|
3
|
+
|
|
4
|
+
Licensed under the MIT License. See LICENSE file at the project root or
|
|
5
|
+
https://opensource.org/license/mit/ for details.
|
|
6
|
+
|
|
7
|
+
SPDX-License-Identifier: MIT
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/*
|
|
11
|
+
researchAgent.js
|
|
12
|
+
Module for simulating a research agent.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
// IMPORTS
|
|
16
|
+
|
|
17
|
+
const {getLogs} = require('./util');
|
|
18
|
+
require('dotenv').config();
|
|
19
|
+
const httpClient = require('http');
|
|
20
|
+
const httpsClient = require('https');
|
|
21
|
+
|
|
22
|
+
// CONSTANTS
|
|
23
|
+
|
|
24
|
+
const agent = process.env.RESEARCH_AGENT;
|
|
25
|
+
const agentPW = process.env.RESEARCH_AGENT_PW;
|
|
26
|
+
const kilotestHosts = [process.env.LOCAL_KILOTEST_HOST, process.env.DEPLOYED_KILOTEST_HOST];
|
|
27
|
+
// Randomly chosen Kilotest host.
|
|
28
|
+
const kilotestHost = kilotestHosts[Math.random() < 0.5 ? 0 : 1];
|
|
29
|
+
const hostParts = kilotestHost.split(/:\/*/);
|
|
30
|
+
const scheme = hostParts[0];
|
|
31
|
+
const host = hostParts[1];
|
|
32
|
+
const port = hostParts[2] || (scheme === 'https' ? 443 : 80);
|
|
33
|
+
const services = ['reportIssues'];
|
|
34
|
+
// Randomly chosen service.
|
|
35
|
+
const service = services[Math.floor(services.length * Math.random())];
|
|
36
|
+
|
|
37
|
+
// FUNCTIONS
|
|
38
|
+
|
|
39
|
+
// Submits a random research request to a random Kilotest host.
|
|
40
|
+
const requestService = async () => {
|
|
41
|
+
const logs = await getLogs();
|
|
42
|
+
// Randomly chosen log.
|
|
43
|
+
const log = logs[Math.floor(logs.length * Math.random())];
|
|
44
|
+
const specs = log.jobName.split('-').join('/');
|
|
45
|
+
const path = `/api/${agent}/${service}/${specs}`;
|
|
46
|
+
const client = scheme === 'https' ? httpsClient : httpClient;
|
|
47
|
+
// Use its job name in the request path.
|
|
48
|
+
const requestOptions = {
|
|
49
|
+
method: 'POST',
|
|
50
|
+
host,
|
|
51
|
+
port,
|
|
52
|
+
path,
|
|
53
|
+
headers: {
|
|
54
|
+
'content-type': 'application/json; charset=utf-8'
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
console.log(`About to submit ${scheme} request as JSON on port ${port} to ${host}${path}`);
|
|
58
|
+
// Submit a request.
|
|
59
|
+
client.request(requestOptions, response => {
|
|
60
|
+
// Initialize a collection of data from the response.
|
|
61
|
+
const chunks = [];
|
|
62
|
+
response
|
|
63
|
+
// If the response throws an error:
|
|
64
|
+
.on('error', async error => {
|
|
65
|
+
// Report it.
|
|
66
|
+
console.log(error.message);
|
|
67
|
+
})
|
|
68
|
+
// If the response delivers data:
|
|
69
|
+
.on('data', chunk => {
|
|
70
|
+
// Add them to the collection.
|
|
71
|
+
chunks.push(chunk);
|
|
72
|
+
})
|
|
73
|
+
// When the response is completed:
|
|
74
|
+
.on('end', async () => {
|
|
75
|
+
const content = chunks.join('');
|
|
76
|
+
// Output it.
|
|
77
|
+
try {
|
|
78
|
+
console.log(JSON.stringify(JSON.parse(content), null, 2));
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
console.log(error.message);
|
|
82
|
+
console.log(`Response content: ${content || 'No content'}`);
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
})
|
|
86
|
+
// Finish sending the job request.
|
|
87
|
+
.end(JSON.stringify({
|
|
88
|
+
agentPW
|
|
89
|
+
}));
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// EXECUTION
|
|
93
|
+
|
|
94
|
+
// Execute the research agent.
|
|
95
|
+
requestService();
|
package/testOrder/index.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
// IMPORTS
|
|
7
7
|
|
|
8
|
-
const {getJSON, getNowStamp, getRecs, isURL} = require('../util');
|
|
8
|
+
const {getJSON, getNowStamp, getRandomString, getRecs, isURL} = require('../util');
|
|
9
9
|
const fs = require('fs/promises');
|
|
10
10
|
const path = require('path');
|
|
11
11
|
|
|
@@ -20,7 +20,7 @@ exports.answer = async (url, what, authCode) => {
|
|
|
20
20
|
const job = JSON.parse(jobTemplateJSON);
|
|
21
21
|
const nowStamp = getNowStamp();
|
|
22
22
|
// Populate the template with job properties.
|
|
23
|
-
const jobIDSuffix =
|
|
23
|
+
const jobIDSuffix = getRandomString(3);
|
|
24
24
|
const jobName = `${nowStamp}-${jobIDSuffix}`;
|
|
25
25
|
job.id = jobName;
|
|
26
26
|
job.creationTimeStamp = nowStamp;
|
package/util.js
CHANGED
|
@@ -34,12 +34,18 @@ const tools = exports.tools = {
|
|
|
34
34
|
nuVnu: ['Html Checker', 'World Wide Web Consortium'],
|
|
35
35
|
qualWeb: ['QualWeb', 'University of Lisbon'],
|
|
36
36
|
testaro: ['Testaro', 'CVS Health'],
|
|
37
|
-
wax: ['WallyAX', 'Wally'],
|
|
38
37
|
wave: ['WAVE', 'Utah State University'],
|
|
39
38
|
};
|
|
39
|
+
exports.researchAgents = {
|
|
40
|
+
'research-agent': 'Internal Research Agent'
|
|
41
|
+
}
|
|
40
42
|
|
|
41
43
|
// FUNCTIONS
|
|
42
44
|
|
|
45
|
+
// Returns a random string.
|
|
46
|
+
exports.getRandomString = length => {
|
|
47
|
+
return Math.random().toString(36).slice(2, length + 2);
|
|
48
|
+
};
|
|
43
49
|
// Returns whether a report is valid.
|
|
44
50
|
exports.isValidReport = report => {
|
|
45
51
|
// Return whether it has the type and properties required by Kilotest:
|
|
@@ -114,7 +120,7 @@ const getDateString = exports.getDateString = timeStamp => {
|
|
|
114
120
|
return '';
|
|
115
121
|
};
|
|
116
122
|
// Returns the date and time represented by a time stamp.
|
|
117
|
-
const getDateTime = timeStamp => {
|
|
123
|
+
const getDateTime = exports.getDateTime = timeStamp => {
|
|
118
124
|
const dateString
|
|
119
125
|
= `20${timeStamp.slice(0, 2)}-${timeStamp.slice(2, 4)}-${timeStamp.slice(4,6)}T${timeStamp.slice(7,9)}:${timeStamp.slice(9,11)}Z`;
|
|
120
126
|
const dateTime = new Date(dateString);
|
|
@@ -484,7 +490,7 @@ exports.getTextFragmentHref = (text, url) => {
|
|
|
484
490
|
// Return a text-fragment link.
|
|
485
491
|
return `${url}#:~:text=${fragmentList}`;
|
|
486
492
|
};
|
|
487
|
-
// Returns an array of the logs
|
|
493
|
+
// Returns an array of the logs, with job names added, of the non-hidden reports.
|
|
488
494
|
exports.getLogs = async () => {
|
|
489
495
|
// Initialize data on the tested targets.
|
|
490
496
|
const logs = [];
|