@jrpool/kilotest 24.1.1 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/env.example +16 -5
- package/index.js +67 -17
- package/package.json +1 -1
- package/reportIssues/api.js +92 -233
- package/reportIssues/index.js +3 -4
- package/reportIssues/util.js +1 -7
- package/researchAgent.js +95 -0
- package/testOrder/index.js +2 -2
- package/util.js +6 -2
package/env.example
CHANGED
|
@@ -1,12 +1,23 @@
|
|
|
1
|
-
#
|
|
2
|
-
ENVIRONMENT=
|
|
3
|
-
# PROTOCOL
|
|
1
|
+
# ENVIRONMENT=test for a dev/test host, =production for a deployed host.
|
|
2
|
+
ENVIRONMENT=__placeholder__
|
|
3
|
+
# PROTOCOL=http for a local dev/test host or reverse-proxied server, otherwise =https.
|
|
4
4
|
PROTOCOL=__placeholder__
|
|
5
|
-
# AUTH_CODE must be 3 characters long.
|
|
5
|
+
# AUTH_CODE for manager operations; must be 3 characters long.
|
|
6
6
|
AUTH_CODE=__placeholder__
|
|
7
|
+
# Name of Testaro agent.
|
|
7
8
|
TESTARO_AGENT=testaro-agent
|
|
8
|
-
#
|
|
9
|
+
# Testaro agent password (same as NETWATCH_AUTH in the Testaro .env.
|
|
9
10
|
TESTARO_AGENT_PW=__placeholder__
|
|
11
|
+
# Name of research agent (AI agent calling Kilotest as a tool).
|
|
12
|
+
RESEARCH_AGENT=research-agent
|
|
13
|
+
# Research agent password.
|
|
14
|
+
RESEARCH_AGENT_PW=__placeholder__
|
|
15
|
+
# URL of this Kilotest host.
|
|
16
|
+
THIS_KILOTEST_HOST=__placeholder__
|
|
17
|
+
# URL of a dev/test Kilotest host.
|
|
18
|
+
LOCAL_KILOTEST_HOST=http://localhost:3000
|
|
19
|
+
# URL of the deployed Kilotest host.
|
|
20
|
+
DEPLOYED_KILOTEST_HOST=https://kilotest.com
|
|
10
21
|
# Alert email configuration:
|
|
11
22
|
MANAGER_EMAIL=__placeholder__
|
|
12
23
|
ALERT_API_HOST=api.resend.com
|
package/index.js
CHANGED
|
@@ -67,30 +67,43 @@ const claimedPath = path.join(jobsPath, 'claimed');
|
|
|
67
67
|
const failedPath = path.join(jobsPath, 'failed');
|
|
68
68
|
const testaroAgent = process.env.TESTARO_AGENT;
|
|
69
69
|
const testaroAgentPW = process.env.TESTARO_AGENT_PW;
|
|
70
|
+
const researchAgent = process.env.RESEARCH_AGENT;
|
|
71
|
+
const researchAgentPW = process.env.RESEARCH_AGENT_PW;
|
|
70
72
|
// Values that may require alerts.
|
|
71
73
|
const balancePath = path.join(__dirname, 'aiService0Balance.json');
|
|
72
74
|
const WAVE_THRESHOLD = Number(process.env.WAVE_BALANCE_THRESHOLD);
|
|
73
75
|
const AI_SERVICE0_THRESHOLD = Number(process.env.AI_SERVICE0_BALANCE_THRESHOLD);
|
|
74
76
|
const AI_MODEL0_INPUT_PRICE = Number(process.env.AI_MODEL0_INPUT_PRICE);
|
|
75
77
|
const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
|
|
78
|
+
const researchAgents = {
|
|
79
|
+
'research-agent': 'Internal Research Agent'
|
|
80
|
+
}
|
|
76
81
|
|
|
77
82
|
// FUNCTIONS
|
|
78
83
|
|
|
79
|
-
// Serves an error message.
|
|
84
|
+
// Serves or sends an error message.
|
|
80
85
|
const serveError = async (error, response, isHumanUser = true) => {
|
|
81
|
-
console.log(error.message);
|
|
86
|
+
console.log(error.message || 'ERROR');
|
|
82
87
|
if (! response.writableEnded) {
|
|
83
88
|
response.statusCode = 400;
|
|
89
|
+
// If the request is from a human user:
|
|
84
90
|
if (isHumanUser) {
|
|
91
|
+
// Serve an HTML page containing the message property of the error.
|
|
85
92
|
response.setHeader('content-type', 'text/html; charset=utf-8');
|
|
86
93
|
const errorTemplate = await fs.readFile('error.html', 'utf8');
|
|
87
|
-
const errorPage = errorTemplate.replace(/__error__/, error.message);
|
|
94
|
+
const errorPage = errorTemplate.replace(/__error__/, error.message || 'ERROR');
|
|
88
95
|
response.end(errorPage);
|
|
89
|
-
}
|
|
96
|
+
}
|
|
97
|
+
// Otherwise, i.e. if it is from an agent:
|
|
98
|
+
else {
|
|
99
|
+
// Send a JSON response containing the entire error.
|
|
90
100
|
response.setHeader('content-type', 'application/json; charset=utf-8');
|
|
91
|
-
response.end(JSON.stringify({error
|
|
101
|
+
response.end(JSON.stringify({error}));
|
|
92
102
|
}
|
|
93
103
|
}
|
|
104
|
+
else {
|
|
105
|
+
console.log('Cannot send error response because the response has ended.');
|
|
106
|
+
}
|
|
94
107
|
};
|
|
95
108
|
// Checks a report for balances nearing exhaustion.
|
|
96
109
|
const checkBalancesForAlerts = async report => {
|
|
@@ -107,7 +120,6 @@ const checkBalancesForAlerts = async report => {
|
|
|
107
120
|
`Only ${creditsRemaining} WAVE credits remain (3 used per job)`
|
|
108
121
|
);
|
|
109
122
|
}
|
|
110
|
-
// AI service 0.
|
|
111
123
|
const testaroAct = report.acts.find(act => act.type === 'test' && act.which === 'testaro');
|
|
112
124
|
// Get the AI model token usage for the testaro allCaps test.
|
|
113
125
|
const usage = testaroAct?.data?.ruleData?.allCaps?.aiModelUsage;
|
|
@@ -231,7 +243,7 @@ const requestHandler = async (request, response) => {
|
|
|
231
243
|
// Otherwise, i.e. if the request is syntactically invalid:
|
|
232
244
|
else {
|
|
233
245
|
// Report the error.
|
|
234
|
-
await serveError({message: 'Invalid report request'}, response, true);
|
|
246
|
+
await serveError({message: 'ERROR: Invalid report request'}, response, true);
|
|
235
247
|
}
|
|
236
248
|
}
|
|
237
249
|
// Otherwise, if it is for an HTML page other than the home page:
|
|
@@ -258,7 +270,7 @@ const requestHandler = async (request, response) => {
|
|
|
258
270
|
// Otherwise, i.e. if the answer cannot be generated:
|
|
259
271
|
else {
|
|
260
272
|
// Report the error.
|
|
261
|
-
await serveError({message: 'Invalid request'}, response, true);
|
|
273
|
+
await serveError({message: 'ERROR: Invalid request'}, response, true);
|
|
262
274
|
}
|
|
263
275
|
}
|
|
264
276
|
// Otherwise, if it is for a tutorial image:
|
|
@@ -274,7 +286,7 @@ const requestHandler = async (request, response) => {
|
|
|
274
286
|
response.end(img);
|
|
275
287
|
}
|
|
276
288
|
catch (_) {
|
|
277
|
-
await serveError({message: 'Image not found'}, response, true);
|
|
289
|
+
await serveError({message: 'ERROR: Image not found'}, response, true);
|
|
278
290
|
}
|
|
279
291
|
}
|
|
280
292
|
// Otherwise, if it is for the application icon:
|
|
@@ -340,7 +352,7 @@ const requestHandler = async (request, response) => {
|
|
|
340
352
|
// Otherwise, i.e. if the request is invalid:
|
|
341
353
|
else {
|
|
342
354
|
// Report the error.
|
|
343
|
-
await serveError({message: 'Invalid retest recommendation'}, response, true);
|
|
355
|
+
await serveError({message: 'ERROR: Invalid retest recommendation'}, response, true);
|
|
344
356
|
}
|
|
345
357
|
}
|
|
346
358
|
// Otherwise, if it is a test recommendation:
|
|
@@ -368,7 +380,7 @@ const requestHandler = async (request, response) => {
|
|
|
368
380
|
// Otherwise, i.e. if the request is invalid:
|
|
369
381
|
else {
|
|
370
382
|
// Report the error.
|
|
371
|
-
await serveError({message: 'Invalid test recommendation'}, response, true);
|
|
383
|
+
await serveError({message: 'ERROR: Invalid test recommendation'}, response, true);
|
|
372
384
|
}
|
|
373
385
|
}
|
|
374
386
|
// Otherwise, if it is an action on a test or retest recommendation:
|
|
@@ -416,7 +428,7 @@ const requestHandler = async (request, response) => {
|
|
|
416
428
|
// Otherwise, i.e. if the request is invalid:
|
|
417
429
|
else {
|
|
418
430
|
// Report the error.
|
|
419
|
-
await serveError({message: 'Invalid test order'}, response, true);
|
|
431
|
+
await serveError({message: 'ERROR: Invalid test order'}, response, true);
|
|
420
432
|
}
|
|
421
433
|
}
|
|
422
434
|
// Otherwise, if it is a reannotation order:
|
|
@@ -459,10 +471,11 @@ const requestHandler = async (request, response) => {
|
|
|
459
471
|
await serveError({message: answerData.error}, response, true);
|
|
460
472
|
}
|
|
461
473
|
}
|
|
462
|
-
// Otherwise, if it is a request from
|
|
474
|
+
// Otherwise, if it is a request from an agent:
|
|
463
475
|
else if (pageName === 'api') {
|
|
464
|
-
|
|
465
|
-
|
|
476
|
+
// Get the agent ID, the service, and any service specifications from the path.
|
|
477
|
+
const [agentID, service, ... specs] = pageArgs.split('/');
|
|
478
|
+
// If the agent is the authorized Testaro instance and it is authenticated:
|
|
466
479
|
if (agentID === testaroAgent && postData.agentPW === testaroAgentPW) {
|
|
467
480
|
// If the service is job assignment:
|
|
468
481
|
if (service === 'job') {
|
|
@@ -570,9 +583,43 @@ const requestHandler = async (request, response) => {
|
|
|
570
583
|
);
|
|
571
584
|
}
|
|
572
585
|
}
|
|
573
|
-
// Otherwise,
|
|
586
|
+
// Otherwise, if the agent is the authorized research agent and it is authenticated:
|
|
587
|
+
else if (agentID === researchAgent && postData.agentPW === researchAgentPW) {
|
|
588
|
+
// If the service is provision of facts about issues in a report:
|
|
589
|
+
if (service === 'reportIssues') {
|
|
590
|
+
// Get the report identifiers from the path.
|
|
591
|
+
const [timeStamp, jobID] = specs;
|
|
592
|
+
const agentName = researchAgents[agentID];
|
|
593
|
+
const args = [agentName, timeStamp, jobID];
|
|
594
|
+
const reportSpecsBad = await isHidden(timeStamp, jobID);
|
|
595
|
+
// If the report is nonexistent or hidden:
|
|
596
|
+
if (reportSpecsBad) {
|
|
597
|
+
// Report this.
|
|
598
|
+
await serveError(
|
|
599
|
+
{message: reportSpecsBad === true ? 'Report nonexistent or hidden' : reportSpecsBad},
|
|
600
|
+
response,
|
|
601
|
+
false
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
// Otherwise, i.e. if the report is available:
|
|
605
|
+
else {
|
|
606
|
+
// Get the response (potentially error) data.
|
|
607
|
+
const responseData = await require(path.join(__dirname, 'reportIssues', 'api'))
|
|
608
|
+
.response(args);
|
|
609
|
+
// Send them.
|
|
610
|
+
response.end(JSON.stringify(responseData));
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
// Otherwise, i.e. if the service is invalid:
|
|
614
|
+
else {
|
|
615
|
+
// Report this.
|
|
616
|
+
await serveError({message: 'ERROR: Invalid service request'}, response, false);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
// Otherwise, i.e. if the agent is not authorized or not authenticated:
|
|
574
620
|
else {
|
|
575
|
-
|
|
621
|
+
// Report this.
|
|
622
|
+
await serveError({message: 'ERROR: Invalid agent'}, response, false);
|
|
576
623
|
}
|
|
577
624
|
}
|
|
578
625
|
// Otherwise, if it is a tutorial comment:
|
|
@@ -617,6 +664,9 @@ const serve = async (protocolModule, options) => {
|
|
|
617
664
|
console.log(`Kilotest server listening at ${protocol}://localhost:${port}.`);
|
|
618
665
|
});
|
|
619
666
|
};
|
|
667
|
+
|
|
668
|
+
// EXECUTION
|
|
669
|
+
|
|
620
670
|
if (protocol === 'http') {
|
|
621
671
|
console.log('Starting HTTP server');
|
|
622
672
|
serve(http, {});
|
package/package.json
CHANGED
package/reportIssues/api.js
CHANGED
|
@@ -1,231 +1,53 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
api.js
|
|
3
|
+
Responds to the report-issues request.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// IMPORTS
|
|
7
7
|
|
|
8
|
+
const {getData, getToolData} = require('./util');
|
|
8
9
|
const {
|
|
9
|
-
|
|
10
|
-
|
|
10
|
+
getDateTime,
|
|
11
|
+
getNowStamp,
|
|
12
|
+
getRandomString,
|
|
11
13
|
getToolNamesString,
|
|
12
|
-
getWCAGLink,
|
|
13
|
-
getWeightName,
|
|
14
|
-
htmlSafe,
|
|
15
14
|
isHidden,
|
|
16
|
-
isValidReport,
|
|
17
|
-
objectSort,
|
|
18
15
|
tools
|
|
19
16
|
} = require('../util');
|
|
20
|
-
const {issues} = require('testilo/procs/score/tic');
|
|
21
|
-
const fs = require('fs/promises');
|
|
22
|
-
const path = require('path');
|
|
23
17
|
|
|
24
18
|
// FUNCTIONS
|
|
25
19
|
|
|
26
|
-
// Gets
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
reportersString: '',
|
|
36
|
-
violators: new Set(),
|
|
37
|
-
violatorCount: 0,
|
|
38
|
-
preventions: {},
|
|
39
|
-
issuesObject: {},
|
|
40
|
-
issueCount: 0,
|
|
41
|
-
issues: []
|
|
20
|
+
// Gets facts about tools.
|
|
21
|
+
const getToolFacts = toolIDs => {
|
|
22
|
+
const crypticData = getToolData(toolIDs);
|
|
23
|
+
return crypticData.map(tool => {
|
|
24
|
+
const {toolID, toolName, toolMaker} = tool;
|
|
25
|
+
return {
|
|
26
|
+
identifier: toolID,
|
|
27
|
+
name: toolName,
|
|
28
|
+
sponsor: toolMaker
|
|
42
29
|
};
|
|
43
|
-
const {issuesObject, reporters, violators} = issuesData;
|
|
44
|
-
// For each act in it:
|
|
45
|
-
report.acts.forEach(act => {
|
|
46
|
-
// If it is a test act:
|
|
47
|
-
if (act.type === 'test') {
|
|
48
|
-
const {result, which} = act;
|
|
49
|
-
const instances = result?.standardResult?.instances ?? [];
|
|
50
|
-
// For each of its standard instances:
|
|
51
|
-
instances.forEach(instance => {
|
|
52
|
-
const {issueID} = instance;
|
|
53
|
-
// If the instance has a non-ignorable classified issue:
|
|
54
|
-
if (issueID && issues[issueID] && issueID !== 'ignorable') {
|
|
55
|
-
// Ensure that the issues data include data on the issue.
|
|
56
|
-
issuesObject[issueID] ??= {
|
|
57
|
-
issueID,
|
|
58
|
-
weight: issues[issueID].weight ?? 0,
|
|
59
|
-
reporters: new Set(),
|
|
60
|
-
reporterCount: 0,
|
|
61
|
-
reportersString: '',
|
|
62
|
-
violators: new Set(),
|
|
63
|
-
violatorCount: 0
|
|
64
|
-
};
|
|
65
|
-
// Ensure that the tool is in the issues data.
|
|
66
|
-
reporters.add(which);
|
|
67
|
-
// Ensure that it is in the issue data.
|
|
68
|
-
issuesObject[issueID].reporters.add(which);
|
|
69
|
-
const {catalogIndex} = instance;
|
|
70
|
-
// If the instance has a catalog index:
|
|
71
|
-
if (catalogIndex) {
|
|
72
|
-
// Ensure that the violator is in the issues data.
|
|
73
|
-
violators.add(catalogIndex);
|
|
74
|
-
// Ensure that it is in the issue data.
|
|
75
|
-
issuesObject[issueID].violators.add(catalogIndex);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
});
|
|
81
|
-
// Populate the unpopulated subproperties of the issues data.
|
|
82
|
-
issuesData.reporterCount = issuesData.reporters.size;
|
|
83
|
-
issuesData.reportersString = getToolNamesString(issuesData.reporters);
|
|
84
|
-
issuesData.violatorCount = issuesData.violators.size;
|
|
85
|
-
issuesData.preventions = report.jobData.preventions;
|
|
86
|
-
issuesData.issueCount = Object.keys(issuesData.issuesObject).length;
|
|
87
|
-
issuesData.issues = Object.values(issuesData.issuesObject);
|
|
88
|
-
// For each issue in the issues data:
|
|
89
|
-
issuesData.issues.forEach(issue => {
|
|
90
|
-
// Populate its unpopulated properties.
|
|
91
|
-
issue.reporterCount = issue.reporters.size;
|
|
92
|
-
issue.reportersString = getToolNamesString(issue.reporters);
|
|
93
|
-
issue.violatorCount = issue.violators.size;
|
|
94
|
-
});
|
|
95
|
-
// Sort the issues alphabetically by reporters string.
|
|
96
|
-
objectSort(issuesData.issues, 'reportersString', 'alpha');
|
|
97
|
-
// Sort the issues again in descending reporter-count order, making this the primary order.
|
|
98
|
-
objectSort(issuesData.issues, 'reporterCount', 'numericDown');
|
|
99
|
-
// Return the issues data.
|
|
100
|
-
return issuesData;
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
// Adds parameters to a query for the answer page.
|
|
104
|
-
const populateQuery = async (timeStamp, jobID, query) => {
|
|
105
|
-
// Get fact descriptions for the report.
|
|
106
|
-
const pageDataStrings = await getPageDataStrings(timeStamp, jobID);
|
|
107
|
-
const {what, urlLink, testInfo} = pageDataStrings;
|
|
108
|
-
const issuesData = await getIssuesData(timeStamp, jobID);
|
|
109
|
-
// If this failed:
|
|
110
|
-
if (typeof issuesData === 'string') {
|
|
111
|
-
// Return this.
|
|
112
|
-
return issuesData;
|
|
113
|
-
}
|
|
114
|
-
const {
|
|
115
|
-
issueCount,
|
|
116
|
-
preventions,
|
|
117
|
-
reporterCount,
|
|
118
|
-
reportersString,
|
|
119
|
-
violatorCount
|
|
120
|
-
} = issuesData;
|
|
121
|
-
// Add an issue count description to the query.
|
|
122
|
-
query.issueCount = issueCount === 1 ? '1 issue was' : `${issueCount} issues were`;
|
|
123
|
-
query.reporterCount = reporterCount === 1 ? '1 tool' : `${reporterCount} tools`;
|
|
124
|
-
// Add a reporter count and list to the query.
|
|
125
|
-
query.reporters = reportersString;
|
|
126
|
-
// Add a violator count to the query.
|
|
127
|
-
query.violatorCount = violatorCount === 1 ? '1 violator was' : `${violatorCount} violators were`;
|
|
128
|
-
// Add page data to the query.
|
|
129
|
-
query.target = what;
|
|
130
|
-
query.urlLink = urlLink;
|
|
131
|
-
query.testInfo = testInfo;
|
|
132
|
-
query.timeStamp = timeStamp;
|
|
133
|
-
query.jobID = jobID;
|
|
134
|
-
const preventionStrings = [];
|
|
135
|
-
const margin = ' '.repeat(6);
|
|
136
|
-
Object.keys(preventions).forEach(preventedToolID => {
|
|
137
|
-
const toolName = tools[preventedToolID];
|
|
138
|
-
const toolNameString = `${toolName[0]} (${toolName[1]})`;
|
|
139
|
-
const causeString = htmlSafe(preventions[preventedToolID]);
|
|
140
|
-
const preventionString = `${margin}<li>Page not testable by ${toolNameString}: ${causeString}</li>`;
|
|
141
|
-
preventionStrings.push(preventionString);
|
|
142
|
-
});
|
|
143
|
-
query.preventions = preventionStrings.join('\n');
|
|
144
|
-
// For each weight:
|
|
145
|
-
[4, 3, 2, 1].forEach(weight => {
|
|
146
|
-
// Initialize data on issues having the weight.
|
|
147
|
-
const weightData = [];
|
|
148
|
-
// Initialize the lines for the weight.
|
|
149
|
-
const weightLines = [];
|
|
150
|
-
// For each issue:
|
|
151
|
-
issuesData.issues.forEach(issueData => {
|
|
152
|
-
const {
|
|
153
|
-
issueID, reporterCount, reportersString, violatorCount, weight: issueWeight
|
|
154
|
-
} = issueData;
|
|
155
|
-
// If it has the weight:
|
|
156
|
-
if (issueWeight === weight) {
|
|
157
|
-
const issue = issues[issueID];
|
|
158
|
-
const {wcag, why} = issue;
|
|
159
|
-
const wcagLink = `<a href="${getWCAGLink(wcag)}">${wcag}</a>`;
|
|
160
|
-
// Add data on it to the weight data.
|
|
161
|
-
weightData.push({
|
|
162
|
-
issueID,
|
|
163
|
-
summary: issue.summary,
|
|
164
|
-
why,
|
|
165
|
-
wcag: wcagLink,
|
|
166
|
-
reporterCount,
|
|
167
|
-
reportersString,
|
|
168
|
-
violatorCount
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
});
|
|
172
|
-
const weightName = getWeightName(weight);
|
|
173
|
-
// Add the issue count to the query.
|
|
174
|
-
query[`${weightName}Count`] = weightData.length;
|
|
175
|
-
// If any reported issues have the weight:
|
|
176
|
-
if (weightData.length) {
|
|
177
|
-
// Add the start of a list of the issues with the weight to the lines.
|
|
178
|
-
weightLines.push(`${margin}<ul class="headed">`);
|
|
179
|
-
// For each issue with the weight:
|
|
180
|
-
weightData.forEach(weightIssue => {
|
|
181
|
-
const {issueID, reporterCount, reportersString, violatorCount, wcag, why} = weightIssue;
|
|
182
|
-
// Add the start of a list item to the lines.
|
|
183
|
-
weightLines.push(`${margin} <li>`);
|
|
184
|
-
// Add a heading summarizing the issue to the lines.
|
|
185
|
-
weightLines.push(`${margin} <h5>${weightIssue.summary}</h5>`);
|
|
186
|
-
// Add the start of alist of facts about the issue to the lines.
|
|
187
|
-
weightLines.push(`${margin} <ul class="pseudoTopLevel">`);
|
|
188
|
-
// Add the issue facts to the lines.
|
|
189
|
-
weightLines.push(`${margin} <li>Why it matters: ${why}`);
|
|
190
|
-
weightLines.push(`${margin} <li>Related WCAG standard: ${wcag}`);
|
|
191
|
-
const reporterCountString = reporterCount === 1 ? '1 tool' : `${reporterCount} tools`;
|
|
192
|
-
weightLines.push(
|
|
193
|
-
`${margin} <li>Reported by ${reporterCountString} (${reportersString})</li>`
|
|
194
|
-
);
|
|
195
|
-
const violatorCountString = violatorCount === 1
|
|
196
|
-
? '1 violator was'
|
|
197
|
-
: `${violatorCount} violators were`;
|
|
198
|
-
weightLines.push(`${margin} <li>${violatorCountString} reported</li>`);
|
|
199
|
-
// Add the end of the fact list to the lines.
|
|
200
|
-
weightLines.push(`${margin} </ul>`);
|
|
201
|
-
// Add the start of a link list to the lines.
|
|
202
|
-
weightLines.push(`${margin} <ul class="nav">`);
|
|
203
|
-
const whereQuestionString = 'Where was the issue found?';
|
|
204
|
-
const labelString = `Where was the ${weightIssue.summary} issue found on the ${what} page?`;
|
|
205
|
-
const href = `href="/reportIssue.html/${issueID}/${timeStamp}/${jobID}"`;
|
|
206
|
-
const label = `aria-label="${labelString}"`;
|
|
207
|
-
const whereLink = `<a ${href} ${label}>${whereQuestionString}</a>`;
|
|
208
|
-
// Add a violations link to the lines.
|
|
209
|
-
weightLines.push(`${margin} <li>${whereLink}</li>`);
|
|
210
|
-
// Add the end of the link list to the lines.
|
|
211
|
-
weightLines.push(`${margin} </ul>`);
|
|
212
|
-
// Add the end of the list item to the lines.
|
|
213
|
-
weightLines.push(`${margin} </li>`);
|
|
214
|
-
});
|
|
215
|
-
// Add the end of the list of issues with the weight to the lines.
|
|
216
|
-
weightLines.push(`${margin}</ul>`);
|
|
217
|
-
// Add the lines documenting the issues with the weight to the query.
|
|
218
|
-
query[`${weightName}Details`] = weightLines.join('\n');
|
|
219
|
-
}
|
|
220
|
-
// Otherwise, i.e. if no reported issues have the weight:
|
|
221
|
-
else {
|
|
222
|
-
query[`${weightName}Details`] = `${margin} <p>None</p>`;
|
|
223
|
-
}
|
|
224
30
|
});
|
|
225
31
|
};
|
|
226
|
-
//
|
|
227
|
-
|
|
228
|
-
const
|
|
32
|
+
// Gets facts about an issue.
|
|
33
|
+
const getIssueFacts = issue => {
|
|
34
|
+
const {issueID, reporterCount, reporterList, reporters, summary, violatorCount, wcag, why} = issue;
|
|
35
|
+
return {
|
|
36
|
+
identifier: issueID,
|
|
37
|
+
summary,
|
|
38
|
+
'related WCAG principle (n.n) or success criterion (n.n.n)': wcag,
|
|
39
|
+
'impact on a user': why,
|
|
40
|
+
'tools reporting violations of rules belonging to the issue': {
|
|
41
|
+
'number of the tools': reporterCount,
|
|
42
|
+
'alphabetized list of names of the tools': reporterList,
|
|
43
|
+
'facts about the tools alphabetized by name': getToolFacts(reporters.map(tool => tool.toolID))
|
|
44
|
+
},
|
|
45
|
+
'number of HTML elements reported as exhibiting the issue': violatorCount
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
// Returns a response to a target-issues request.
|
|
49
|
+
exports.response = async args => {
|
|
50
|
+
const [agentName, timeStamp, jobID] = args;
|
|
229
51
|
const reportIsHidden = await isHidden(timeStamp, jobID);
|
|
230
52
|
// If the report is not available:
|
|
231
53
|
if (reportIsHidden) {
|
|
@@ -234,26 +56,63 @@ exports.answer = async pageArgs => {
|
|
|
234
56
|
message: 'Report not available'
|
|
235
57
|
};
|
|
236
58
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
59
|
+
// Otherwise, i.e. if the report is available, get data on the target and issues.
|
|
60
|
+
const data = await getData(timeStamp, jobID);
|
|
61
|
+
const {pageData, issuesData} = data;
|
|
62
|
+
const {what, url, daysAgo} = pageData;
|
|
63
|
+
const {issueCount, issues, preventions, reporterCount, reporters, violatorCount} = issuesData;
|
|
64
|
+
const preventedTools = getToolFacts(Object.keys(preventions));
|
|
65
|
+
preventedTools.forEach(preventedTool => {
|
|
66
|
+
preventedTool['reason for failure'] = preventions[preventedTool.identifier];
|
|
67
|
+
});
|
|
68
|
+
const thisHost = process.env.THIS_KILOTEST_HOST;
|
|
69
|
+
// Get a response.
|
|
70
|
+
const response = {
|
|
71
|
+
summary: `This document fulfills a request made by an agent to Kilotest. The agent requested data from a report produced by Kilotest. The report contains results of tests performed on a web page by an ensemble of tools. The tools use a combination of rule-based and machine-learning-based methods to identify accessibility, usability, and standard-conformity issues. More detailed information about Kilotest is available from its deployed instance, ${process.env.DEPLOYED_KILOTEST_HOST}. which contains an introduction on its home page and a tutorial.`,
|
|
72
|
+
'tool name': 'Kilotest',
|
|
73
|
+
request: {
|
|
74
|
+
'name of requesting agent': agentName,
|
|
75
|
+
'type of request': {
|
|
76
|
+
identifier: 'reportIssues',
|
|
77
|
+
description: 'What issues did Kilotest report in the specified report?'
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
'response metadata': {
|
|
81
|
+
'date and time': new Date().toISOString(),
|
|
82
|
+
'identifier': `${getNowStamp()}-${getRandomString(3)}`
|
|
83
|
+
},
|
|
84
|
+
report: {
|
|
85
|
+
identifier: `${timeStamp}-${jobID}`,
|
|
86
|
+
'creation date': getDateTime(timeStamp),
|
|
87
|
+
'days since the creation date': daysAgo,
|
|
88
|
+
'URL for human inspection': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
|
|
89
|
+
},
|
|
90
|
+
'tested web page': {
|
|
91
|
+
description: what,
|
|
92
|
+
URL: url
|
|
93
|
+
},
|
|
94
|
+
'names of tools that tried to test the page': getToolNamesString(Object.keys(tools)),
|
|
95
|
+
'tools that were unable to test the page': preventedTools,
|
|
96
|
+
'tools that reported issues': {
|
|
97
|
+
'number of tools': reporterCount,
|
|
98
|
+
'facts about the tools': getToolFacts(reporters.map(tool => tool.toolID))
|
|
99
|
+
},
|
|
100
|
+
'number of issues reported': {
|
|
101
|
+
'total': issueCount,
|
|
102
|
+
'by priority': {
|
|
103
|
+
'highest priority': issues[4].length,
|
|
104
|
+
'high priority': issues[3].length,
|
|
105
|
+
'low priority': issues[2].length,
|
|
106
|
+
'lowest priority': issues[1].length
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
'number of HTML elements reported as exhibiting issues': violatorCount,
|
|
110
|
+
'issues reported': {
|
|
111
|
+
'highest priority': issues[4].map(issue => getIssueFacts(issue)),
|
|
112
|
+
'high priority': issues[3].map(issue => getIssueFacts(issue)),
|
|
113
|
+
'low priority': issues[2].map(issue => getIssueFacts(issue)),
|
|
114
|
+
'lowest priority': issues[1].map(issue => getIssueFacts(issue))
|
|
115
|
+
}
|
|
258
116
|
};
|
|
117
|
+
return response;
|
|
259
118
|
};
|
package/reportIssues/index.js
CHANGED
|
@@ -14,7 +14,6 @@ const {
|
|
|
14
14
|
isHidden,
|
|
15
15
|
tools
|
|
16
16
|
} = require('../util');
|
|
17
|
-
const {issues} = require('testilo/procs/score/tic');
|
|
18
17
|
const fs = require('fs/promises');
|
|
19
18
|
const path = require('path');
|
|
20
19
|
|
|
@@ -22,7 +21,7 @@ const path = require('path');
|
|
|
22
21
|
|
|
23
22
|
// Adds parameters to a query for the answer page.
|
|
24
23
|
const populateQuery = async (timeStamp, jobID, query) => {
|
|
25
|
-
// Get data on the
|
|
24
|
+
// Get data on the target and its issues according to the report.
|
|
26
25
|
const data = await getData(timeStamp, jobID);
|
|
27
26
|
const {pageData, issuesData} = data;
|
|
28
27
|
// If the page data are invalid:
|
|
@@ -35,10 +34,10 @@ const populateQuery = async (timeStamp, jobID, query) => {
|
|
|
35
34
|
// Return this.
|
|
36
35
|
return issuesData;
|
|
37
36
|
}
|
|
38
|
-
// Otherwise, get fact descriptions for the
|
|
37
|
+
// Otherwise, get fact descriptions for the target.
|
|
39
38
|
const pageInfo = await getPageDataStrings(timeStamp, jobID, pageData);
|
|
40
39
|
const {what, urlLink, testInfo} = pageInfo;
|
|
41
|
-
// Add
|
|
40
|
+
// Add target data to the query.
|
|
42
41
|
query.target = what;
|
|
43
42
|
query.urlLink = urlLink;
|
|
44
43
|
query.testInfo = testInfo;
|
package/reportIssues/util.js
CHANGED
|
@@ -11,7 +11,7 @@ const issuesClassification = require('testilo/procs/score/tic').issues;
|
|
|
11
11
|
// FUNCTIONS
|
|
12
12
|
|
|
13
13
|
// Converts tool IDs to tool data sorted by tool name.
|
|
14
|
-
const getToolData = toolIDs => objectSort(
|
|
14
|
+
const getToolData = exports.getToolData = toolIDs => objectSort(
|
|
15
15
|
Array.from(toolIDs).map(toolID => {
|
|
16
16
|
const toolData = tools[toolID];
|
|
17
17
|
return {
|
|
@@ -47,12 +47,6 @@ const getIssuesData = async (timeStamp, jobID) => {
|
|
|
47
47
|
reporterCount: 0,
|
|
48
48
|
violatorCount: 0,
|
|
49
49
|
preventions: report.jobData.preventions,
|
|
50
|
-
priorityNames: {
|
|
51
|
-
4: 'highest',
|
|
52
|
-
3: 'high',
|
|
53
|
-
2: 'low',
|
|
54
|
-
1: 'lowest'
|
|
55
|
-
},
|
|
56
50
|
issues: {
|
|
57
51
|
4: [],
|
|
58
52
|
3: [],
|
package/researchAgent.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/*
|
|
2
|
+
© 2026 Jonathan Robert Pool.
|
|
3
|
+
|
|
4
|
+
Licensed under the MIT License. See LICENSE file at the project root or
|
|
5
|
+
https://opensource.org/license/mit/ for details.
|
|
6
|
+
|
|
7
|
+
SPDX-License-Identifier: MIT
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/*
|
|
11
|
+
researchAgent.js
|
|
12
|
+
Module for simulating a research agent.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
// IMPORTS
|
|
16
|
+
|
|
17
|
+
const {getLogs} = require('./util');
|
|
18
|
+
require('dotenv').config();
|
|
19
|
+
const httpClient = require('http');
|
|
20
|
+
const httpsClient = require('https');
|
|
21
|
+
|
|
22
|
+
// CONSTANTS
|
|
23
|
+
|
|
24
|
+
const agent = process.env.RESEARCH_AGENT;
|
|
25
|
+
const agentPW = process.env.RESEARCH_AGENT_PW;
|
|
26
|
+
const kilotestHosts = [process.env.LOCAL_KILOTEST_HOST, process.env.DEPLOYED_KILOTEST_HOST];
|
|
27
|
+
// Randomly chosen Kilotest host.
|
|
28
|
+
const kilotestHost = kilotestHosts[Math.random() < 0.5 ? 0 : 1];
|
|
29
|
+
const hostParts = kilotestHost.split(/:\/*/);
|
|
30
|
+
const scheme = hostParts[0];
|
|
31
|
+
const host = hostParts[1];
|
|
32
|
+
const port = hostParts[2] || (scheme === 'https' ? 443 : 80);
|
|
33
|
+
const services = ['reportIssues'];
|
|
34
|
+
// Randomly chosen service.
|
|
35
|
+
const service = services[Math.floor(services.length * Math.random())];
|
|
36
|
+
|
|
37
|
+
// FUNCTIONS
|
|
38
|
+
|
|
39
|
+
// Submits a random research request to a random Kilotest host.
|
|
40
|
+
const requestService = async () => {
|
|
41
|
+
const logs = await getLogs();
|
|
42
|
+
// Randomly chosen log.
|
|
43
|
+
const log = logs[Math.floor(logs.length * Math.random())];
|
|
44
|
+
const specs = log.jobName.split('-').join('/');
|
|
45
|
+
const path = `/api/${agent}/${service}/${specs}`;
|
|
46
|
+
const client = scheme === 'https' ? httpsClient : httpClient;
|
|
47
|
+
// Use its job name in the request path.
|
|
48
|
+
const requestOptions = {
|
|
49
|
+
method: 'POST',
|
|
50
|
+
host,
|
|
51
|
+
port,
|
|
52
|
+
path,
|
|
53
|
+
headers: {
|
|
54
|
+
'content-type': 'application/json; charset=utf-8'
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
console.log(`About to submit ${scheme} request as JSON on port ${port} to ${host}${path}`);
|
|
58
|
+
// Submit a request.
|
|
59
|
+
client.request(requestOptions, response => {
|
|
60
|
+
// Initialize a collection of data from the response.
|
|
61
|
+
const chunks = [];
|
|
62
|
+
response
|
|
63
|
+
// If the response throws an error:
|
|
64
|
+
.on('error', async error => {
|
|
65
|
+
// Report it.
|
|
66
|
+
console.log(error.message);
|
|
67
|
+
})
|
|
68
|
+
// If the response delivers data:
|
|
69
|
+
.on('data', chunk => {
|
|
70
|
+
// Add them to the collection.
|
|
71
|
+
chunks.push(chunk);
|
|
72
|
+
})
|
|
73
|
+
// When the response is completed:
|
|
74
|
+
.on('end', async () => {
|
|
75
|
+
const content = chunks.join('');
|
|
76
|
+
// Output it.
|
|
77
|
+
try {
|
|
78
|
+
console.log(JSON.stringify(JSON.parse(content), null, 2));
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
console.log(error.message);
|
|
82
|
+
console.log(`Response content: ${content || 'No content'}`);
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
})
|
|
86
|
+
// Finish sending the job request.
|
|
87
|
+
.end(JSON.stringify({
|
|
88
|
+
agentPW
|
|
89
|
+
}));
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// EXECUTION
|
|
93
|
+
|
|
94
|
+
// Execute the research agent.
|
|
95
|
+
requestService();
|
package/testOrder/index.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
// IMPORTS
|
|
7
7
|
|
|
8
|
-
const {getJSON, getNowStamp, getRecs, isURL} = require('../util');
|
|
8
|
+
const {getJSON, getNowStamp, getRandomString, getRecs, isURL} = require('../util');
|
|
9
9
|
const fs = require('fs/promises');
|
|
10
10
|
const path = require('path');
|
|
11
11
|
|
|
@@ -20,7 +20,7 @@ exports.answer = async (url, what, authCode) => {
|
|
|
20
20
|
const job = JSON.parse(jobTemplateJSON);
|
|
21
21
|
const nowStamp = getNowStamp();
|
|
22
22
|
// Populate the template with job properties.
|
|
23
|
-
const jobIDSuffix =
|
|
23
|
+
const jobIDSuffix = getRandomString(3);
|
|
24
24
|
const jobName = `${nowStamp}-${jobIDSuffix}`;
|
|
25
25
|
job.id = jobName;
|
|
26
26
|
job.creationTimeStamp = nowStamp;
|
package/util.js
CHANGED
|
@@ -40,6 +40,10 @@ const tools = exports.tools = {
|
|
|
40
40
|
|
|
41
41
|
// FUNCTIONS
|
|
42
42
|
|
|
43
|
+
// Returns a random string.
|
|
44
|
+
exports.getRandomString = length => {
|
|
45
|
+
return Math.random().toString(36).slice(2, length + 2);
|
|
46
|
+
};
|
|
43
47
|
// Returns whether a report is valid.
|
|
44
48
|
exports.isValidReport = report => {
|
|
45
49
|
// Return whether it has the type and properties required by Kilotest:
|
|
@@ -114,7 +118,7 @@ const getDateString = exports.getDateString = timeStamp => {
|
|
|
114
118
|
return '';
|
|
115
119
|
};
|
|
116
120
|
// Returns the date and time represented by a time stamp.
|
|
117
|
-
const getDateTime = timeStamp => {
|
|
121
|
+
const getDateTime = exports.getDateTime = timeStamp => {
|
|
118
122
|
const dateString
|
|
119
123
|
= `20${timeStamp.slice(0, 2)}-${timeStamp.slice(2, 4)}-${timeStamp.slice(4,6)}T${timeStamp.slice(7,9)}:${timeStamp.slice(9,11)}Z`;
|
|
120
124
|
const dateTime = new Date(dateString);
|
|
@@ -484,7 +488,7 @@ exports.getTextFragmentHref = (text, url) => {
|
|
|
484
488
|
// Return a text-fragment link.
|
|
485
489
|
return `${url}#:~:text=${fragmentList}`;
|
|
486
490
|
};
|
|
487
|
-
// Returns an array of the logs
|
|
491
|
+
// Returns an array of the logs, with job names added, of the non-hidden reports.
|
|
488
492
|
exports.getLogs = async () => {
|
|
489
493
|
// Initialize data on the tested targets.
|
|
490
494
|
const logs = [];
|