@jrpool/kilotest 38.1.0 → 39.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.markdownlint-cli2.jsonc +12 -0
- package/.vscode/settings.json +3 -0
- package/CLAUDE.md +5 -0
- package/README.md +6 -4
- package/ai0BalanceForm/index.html +1 -1
- package/alerts.js +3 -1
- package/api/getReport.js +66 -0
- package/api/listDiagnoses.js +157 -0
- package/api/listIssues.js +225 -0
- package/api/listReports.js +87 -0
- package/api/listViolators.js +164 -0
- package/api/requestFeature.js +56 -0
- package/api/requestRetest.js +102 -0
- package/api/requestTest.js +106 -0
- package/api/routes.js +97 -0
- package/api/schemas.js +297 -0
- package/api/testSuite.js +294 -0
- package/api/util.js +149 -0
- package/diagnoses/index.js +2 -2
- package/docs/{AI-TOOL.md → AI-TOOLS.md} +54 -26
- package/docs/SERVICE.md +4 -1
- package/docs/decisions/001-api-design.md +59 -49
- package/eslint.config.mjs +13 -3
- package/index.html +1 -1
- package/index.js +202 -188
- package/issues/index.js +13 -21
- package/jsconfig.json +8 -0
- package/llms-full.txt +2 -2
- package/llms.txt +16 -11
- package/mcp.js +129 -65
- package/openapi.yaml +1234 -572
- package/package.json +18 -11
- package/reannotate/index.js +26 -15
- package/reannotateForm/index.js +15 -13
- package/redocly.yaml +4 -0
- package/reportHideForm/index.html +1 -1
- package/reportHideForm/index.js +24 -38
- package/reportIssue/index.js +3 -3
- package/reportIssues/index.js +128 -1
- package/reportUnhideForm/index.js +36 -44
- package/reportsExpungeForm/index.js +22 -16
- package/reportsPruneForm/index.js +22 -12
- package/reportsRewindForm/index.js +17 -9
- package/retestRec/index.js +8 -4
- package/retestRecForm/index.js +18 -6
- package/scripts/generate-openapi.js +83 -0
- package/targets/index.js +18 -13
- package/testOrder/index.js +11 -8
- package/tutorial/index.html +12 -12
- package/util.js +581 -539
- package/api/reportList.js +0 -83
- package/api/reportSummary.js +0 -181
- package/reportIssue/api.js +0 -183
- package/reportIssues/util.js +0 -138
- package/researchAgent.js +0 -206
- package/target/api.js +0 -126
- package/testRecForm/api.js +0 -55
package/index.js
CHANGED
|
@@ -10,21 +10,22 @@ require('dotenv').config({quiet: true});
|
|
|
10
10
|
// IMPORTS
|
|
11
11
|
const {
|
|
12
12
|
annotateReport,
|
|
13
|
+
createLock,
|
|
13
14
|
getJobNames,
|
|
14
15
|
getJSON,
|
|
15
|
-
getLogPath,
|
|
16
16
|
getObject,
|
|
17
17
|
getPOSTData,
|
|
18
18
|
getReport,
|
|
19
19
|
getRecs,
|
|
20
20
|
getReportPath,
|
|
21
|
+
hiddenReportsPath,
|
|
21
22
|
isHidden,
|
|
22
23
|
isReportAvailable,
|
|
23
24
|
isTimeStamp,
|
|
24
25
|
isJobID,
|
|
25
|
-
isURL,
|
|
26
26
|
jobsPath,
|
|
27
|
-
|
|
27
|
+
makeReportsExtract,
|
|
28
|
+
recsLock,
|
|
28
29
|
reportsPath,
|
|
29
30
|
ruleIDs
|
|
30
31
|
} = require('./util');
|
|
@@ -71,7 +72,8 @@ const failedPath = path.join(jobsPath, 'failed');
|
|
|
71
72
|
const testaroAgent = process.env.TESTARO_AGENT;
|
|
72
73
|
const testaroAgentPW = process.env.TESTARO_AGENT_PW;
|
|
73
74
|
// Values that may require alerts.
|
|
74
|
-
const balancePath = path.join(__dirname, '
|
|
75
|
+
const balancePath = path.join(__dirname, 'ai0Balance.json');
|
|
76
|
+
const jobLock = createLock();
|
|
75
77
|
const WAVE_THRESHOLD = Number(process.env.WAVE_BALANCE_THRESHOLD);
|
|
76
78
|
const AI_SERVICE0_THRESHOLD = Number(process.env.AI_SERVICE0_BALANCE_THRESHOLD);
|
|
77
79
|
const AI_MODEL0_INPUT_PRICE = Number(process.env.AI_MODEL0_INPUT_PRICE);
|
|
@@ -83,7 +85,7 @@ const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
|
|
|
83
85
|
const serveError = async (error, response, isHumanUser = true) => {
|
|
84
86
|
const errorLines = Object.entries(error).map(pair => `${pair[0]}: ${pair[1]}`);
|
|
85
87
|
console.log(errorLines.join('\n') || 'ERROR');
|
|
86
|
-
if (!
|
|
88
|
+
if (!response.writableEnded) {
|
|
87
89
|
response.statusCode = 400;
|
|
88
90
|
// If the request is from a human user:
|
|
89
91
|
if (isHumanUser) {
|
|
@@ -152,6 +154,7 @@ const checkBalancesForAlerts = async report => {
|
|
|
152
154
|
if (typeof newBalance === 'number') {
|
|
153
155
|
// Update the recorded balance.
|
|
154
156
|
await fs.writeFile(balancePath, getJSON({balance: newBalance}));
|
|
157
|
+
console.log(`Estimated new AI Service 0 balance: $${newBalance.toFixed(2)}`);
|
|
155
158
|
// If it is nearing exhaustion:
|
|
156
159
|
if (newBalance < AI_SERVICE0_THRESHOLD) {
|
|
157
160
|
// Alert a manager.
|
|
@@ -186,6 +189,71 @@ const getAbuseError = (request, reason) => {
|
|
|
186
189
|
time: new Date().toISOString()
|
|
187
190
|
};
|
|
188
191
|
};
|
|
192
|
+
// Processes a job request from a Testaro agent.
|
|
193
|
+
const processJobRequest = async (request, response, agentID) => jobLock(async () => {
|
|
194
|
+
let clean = true;
|
|
195
|
+
const messageStart = `Testaro agent ${agentID} requested a job, `;
|
|
196
|
+
const jobNames = await getJobNames();
|
|
197
|
+
const claimedJobNames = jobNames.claimed;
|
|
198
|
+
// For each claimed job:
|
|
199
|
+
for (const jobName of claimedJobNames) {
|
|
200
|
+
const job = await getObject(path.join(jobsPath, 'claimed', jobName));
|
|
201
|
+
const {id, sources} = job;
|
|
202
|
+
const {agent} = sources;
|
|
203
|
+
// If its assignee is the agent:
|
|
204
|
+
if (agent === agentID) {
|
|
205
|
+
const messageEnd = `but has not completed job ${id}`;
|
|
206
|
+
// Report this.
|
|
207
|
+
await serveError({message: `${messageStart}${messageEnd}`}, response, false);
|
|
208
|
+
// Reclassify the job as failed.
|
|
209
|
+
await fs.rename(
|
|
210
|
+
path.join(claimedPath, jobName), path.join(failedPath, jobName)
|
|
211
|
+
);
|
|
212
|
+
clean = false;
|
|
213
|
+
// Stop checking claimed jobs.
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
// If no aborted-job error was found for the agent:
|
|
218
|
+
if (clean) {
|
|
219
|
+
const queuedJobNames = jobNames.queue;
|
|
220
|
+
// If any jobs are queued:
|
|
221
|
+
if (queuedJobNames.length) {
|
|
222
|
+
const oldestJobName = queuedJobNames[0];
|
|
223
|
+
// Get the first one.
|
|
224
|
+
const firstJob = await getObject(path.join(queuePath, oldestJobName));
|
|
225
|
+
// Add the agent ID to the job.
|
|
226
|
+
firstJob.sources.agent = agentID;
|
|
227
|
+
console.log(
|
|
228
|
+
`Job ${firstJob.id} (${firstJob.target.what}) is being sent to the agent.`
|
|
229
|
+
);
|
|
230
|
+
// Assign the job to the agent.
|
|
231
|
+
response.writeHead(200, {
|
|
232
|
+
'content-type': 'application/json; charset=utf-8'
|
|
233
|
+
});
|
|
234
|
+
response.end(JSON.stringify(firstJob));
|
|
235
|
+
const messageEnd
|
|
236
|
+
= `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the agent`;
|
|
237
|
+
console.log(`${messageStart}${messageEnd}`);
|
|
238
|
+
// Save the job in the claimed-jobs directory.
|
|
239
|
+
await fs.writeFile(
|
|
240
|
+
path.join(claimedPath, oldestJobName), getJSON(firstJob)
|
|
241
|
+
);
|
|
242
|
+
// Delete it from the queue.
|
|
243
|
+
await fs.unlink(path.join(queuePath, oldestJobName));
|
|
244
|
+
}
|
|
245
|
+
// Otherwise, i.e. if no jobs are queued:
|
|
246
|
+
else {
|
|
247
|
+
response.writeHead(200, {
|
|
248
|
+
'content-type': 'application/json; charset=utf-8'
|
|
249
|
+
});
|
|
250
|
+
// Send a no-jobs response to the agent.
|
|
251
|
+
response.end(JSON.stringify({}));
|
|
252
|
+
const messageEnd = 'but no job was in the queue';
|
|
253
|
+
console.log(`${messageStart}${messageEnd}`);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
});
|
|
189
257
|
// Handles a request.
|
|
190
258
|
const requestHandler = async (request, response) => {
|
|
191
259
|
// Sets response headers.
|
|
@@ -319,7 +387,7 @@ const requestHandler = async (request, response) => {
|
|
|
319
387
|
const topic = pageName.slice(0, -5);
|
|
320
388
|
// If the page can be generated:
|
|
321
389
|
if (answer[topic]) {
|
|
322
|
-
setHeaders('text/html',
|
|
390
|
+
setHeaders('text/html', pathname, 'ultra');
|
|
323
391
|
// Get the answer data.
|
|
324
392
|
const answerData = await answer[topic](pathTail, search);
|
|
325
393
|
// If they are valid:
|
|
@@ -341,37 +409,56 @@ const requestHandler = async (request, response) => {
|
|
|
341
409
|
}
|
|
342
410
|
// Otherwise, if it is for an API service:
|
|
343
411
|
else if (pageName === 'api') {
|
|
344
|
-
const [service,
|
|
345
|
-
// If the service
|
|
346
|
-
if (service === '
|
|
347
|
-
// Get the response
|
|
348
|
-
const
|
|
349
|
-
|
|
412
|
+
const [service, ...specs] = pathTail.split('/');
|
|
413
|
+
// If the service lists the available reports:
|
|
414
|
+
if (service === 'listReports') {
|
|
415
|
+
// Get the response body.
|
|
416
|
+
const responseBody = await require(path.join(__dirname, 'api', 'listReports'))
|
|
417
|
+
.response(specs);
|
|
418
|
+
// Send it.
|
|
350
419
|
setHeaders('application/json', null, 'ultra');
|
|
351
|
-
response.end(JSON.stringify(
|
|
420
|
+
response.end(JSON.stringify(responseBody));
|
|
352
421
|
}
|
|
353
|
-
// Otherwise, if the service
|
|
354
|
-
else if (service === '
|
|
355
|
-
// Get the response
|
|
356
|
-
const
|
|
422
|
+
// Otherwise, if the service lists the issues in a report:
|
|
423
|
+
else if (service === 'listIssues') {
|
|
424
|
+
// Get the response body.
|
|
425
|
+
const responseBody = await require(path.join(__dirname, 'api', 'listIssues'))
|
|
357
426
|
.response(specs);
|
|
358
|
-
// Send
|
|
427
|
+
// Send it.
|
|
359
428
|
setHeaders('application/json', null, 'high');
|
|
360
|
-
response.end(JSON.stringify(
|
|
429
|
+
response.end(JSON.stringify(responseBody));
|
|
361
430
|
}
|
|
362
|
-
// Otherwise, if the service
|
|
363
|
-
else if (service === '
|
|
364
|
-
// Get the response
|
|
365
|
-
const
|
|
431
|
+
// Otherwise, if the service lists the violators of an issue in a report:
|
|
432
|
+
else if (service === 'listViolators') {
|
|
433
|
+
// Get the response body.
|
|
434
|
+
const responseBody = await require(path.join(__dirname, 'api', 'listViolators'))
|
|
366
435
|
.response(specs);
|
|
367
|
-
// Send
|
|
436
|
+
// Send it.
|
|
368
437
|
setHeaders('application/json', null, 'high');
|
|
369
|
-
response.end(JSON.stringify(
|
|
438
|
+
response.end(JSON.stringify(responseBody));
|
|
439
|
+
}
|
|
440
|
+
// Otherwise, if the service lists the diagnoses of a violation of an issue in a report:
|
|
441
|
+
else if (service === 'listDiagnoses') {
|
|
442
|
+
// Get the response body.
|
|
443
|
+
const responseBody = await require(path.join(__dirname, 'api', 'listDiagnoses'))
|
|
444
|
+
.response(specs);
|
|
445
|
+
// Send it.
|
|
446
|
+
setHeaders('application/json', null, 'high');
|
|
447
|
+
response.end(JSON.stringify(responseBody));
|
|
448
|
+
}
|
|
449
|
+
// Otherwise, if the service serves a report:
|
|
450
|
+
else if (service === 'getReport') {
|
|
451
|
+
// Get the response body.
|
|
452
|
+
const responseBody = await require(path.join(__dirname, 'api', 'getReport'))
|
|
453
|
+
.response(specs);
|
|
454
|
+
// Send it.
|
|
455
|
+
setHeaders('application/json', null, 'low');
|
|
456
|
+
response.end(JSON.stringify(responseBody));
|
|
370
457
|
}
|
|
371
458
|
// Otherwise, i.e. if the service is invalid:
|
|
372
459
|
else {
|
|
373
460
|
// Report this.
|
|
374
|
-
await serveError({message: '
|
|
461
|
+
await serveError({message: 'Invalid service request'}, response, false);
|
|
375
462
|
}
|
|
376
463
|
}
|
|
377
464
|
// Otherwise, if it is for a tutorial image:
|
|
@@ -421,7 +508,7 @@ const requestHandler = async (request, response) => {
|
|
|
421
508
|
else {
|
|
422
509
|
// Report the error.
|
|
423
510
|
await serveError(
|
|
424
|
-
{message: `ERROR: Invalid GET request (${pathname}
|
|
511
|
+
{message: `ERROR: Invalid GET request (${pathname})`}, response, true
|
|
425
512
|
);
|
|
426
513
|
}
|
|
427
514
|
}
|
|
@@ -435,36 +522,8 @@ const requestHandler = async (request, response) => {
|
|
|
435
522
|
else {
|
|
436
523
|
// Get the data from the request body.
|
|
437
524
|
const postData = await getPOSTData(request);
|
|
438
|
-
// If the request is a
|
|
439
|
-
if (pageName === '
|
|
440
|
-
const {why} = postData;
|
|
441
|
-
const [timeStamp, jobID] = pathTail.split('/');
|
|
442
|
-
// If the request is valid:
|
|
443
|
-
if (isTimeStamp(timeStamp) && isJobID(jobID) && why) {
|
|
444
|
-
// Serve response headers.
|
|
445
|
-
setHeaders('text/html', `${pathname}${search}`, 'ultra');
|
|
446
|
-
// Get the answer data.
|
|
447
|
-
const answerData = await require(path.join(__dirname, 'retestRec', 'index'))
|
|
448
|
-
.answer(pathTail, why);
|
|
449
|
-
// If they are valid:
|
|
450
|
-
if (answerData.status === 'ok') {
|
|
451
|
-
// Serve the answer page.
|
|
452
|
-
response.end(answerData.answerPage);
|
|
453
|
-
}
|
|
454
|
-
// Otherwise, i.e. if they are invalid:
|
|
455
|
-
else {
|
|
456
|
-
// Report the error.
|
|
457
|
-
await serveError({message: answerData.message}, response, true);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
// Otherwise, i.e. if the request is invalid:
|
|
461
|
-
else {
|
|
462
|
-
// Report the error.
|
|
463
|
-
await serveError({message: 'ERROR: Invalid retest recommendation'}, response, true);
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
// Otherwise, if it is a test recommendation:
|
|
467
|
-
else if (pageName === 'testRec.html') {
|
|
525
|
+
// If the request is a test recommendation:
|
|
526
|
+
if (pageName === 'testRec.html') {
|
|
468
527
|
const {what, url, why} = postData;
|
|
469
528
|
// If the request is valid:
|
|
470
529
|
if (what && url.startsWith('https://') && why) {
|
|
@@ -476,7 +535,7 @@ const requestHandler = async (request, response) => {
|
|
|
476
535
|
// Otherwise, i.e. if no report on the page is available:
|
|
477
536
|
else {
|
|
478
537
|
// Serve headers for a response.
|
|
479
|
-
setHeaders('text/html',
|
|
538
|
+
setHeaders('text/html', pathname, 'ultra');
|
|
480
539
|
// Get the answer data.
|
|
481
540
|
const answerData = await require(path.join(__dirname, 'testRec', 'index'))
|
|
482
541
|
.answer(what, url, why);
|
|
@@ -498,6 +557,34 @@ const requestHandler = async (request, response) => {
|
|
|
498
557
|
await serveError({message: 'ERROR: Invalid test recommendation'}, response, true);
|
|
499
558
|
}
|
|
500
559
|
}
|
|
560
|
+
// Otherwise, if it is a retest recommendation:
|
|
561
|
+
else if (pageName === 'retestRec.html') {
|
|
562
|
+
const {why} = postData;
|
|
563
|
+
const [timeStamp, jobID] = pathTail.split('/');
|
|
564
|
+
// If the request is valid:
|
|
565
|
+
if (isTimeStamp(timeStamp) && isJobID(jobID) && why) {
|
|
566
|
+
// Serve response headers.
|
|
567
|
+
setHeaders('text/html', pathname, 'ultra');
|
|
568
|
+
// Get the answer data.
|
|
569
|
+
const answerData = await require(path.join(__dirname, 'retestRec', 'index'))
|
|
570
|
+
.answer(pathTail, why);
|
|
571
|
+
// If they are valid:
|
|
572
|
+
if (answerData.status === 'ok') {
|
|
573
|
+
// Serve the answer page.
|
|
574
|
+
response.end(answerData.answerPage);
|
|
575
|
+
}
|
|
576
|
+
// Otherwise, i.e. if they are invalid:
|
|
577
|
+
else {
|
|
578
|
+
// Report the error.
|
|
579
|
+
await serveError({message: answerData.message}, response, true);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
// Otherwise, i.e. if the request is invalid:
|
|
583
|
+
else {
|
|
584
|
+
// Report the error.
|
|
585
|
+
await serveError({message: 'ERROR: Invalid retest recommendation'}, response, true);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
501
588
|
// Otherwise, if it is an action on a test or retest recommendation:
|
|
502
589
|
else if (pageName === 'recAction.html') {
|
|
503
590
|
const {target, authCode} = postData;
|
|
@@ -509,8 +596,8 @@ const requestHandler = async (request, response) => {
|
|
|
509
596
|
// If the request is an approval:
|
|
510
597
|
if (what) {
|
|
511
598
|
// Set a location header for a response.
|
|
512
|
-
response.setHeader('content-location',
|
|
513
|
-
//
|
|
599
|
+
response.setHeader('content-location', pathname);
|
|
600
|
+
// Process the approval and get the answer data about the remaining recommendations.
|
|
514
601
|
const answerData = await require(path.join(__dirname, 'testOrder', 'index'))
|
|
515
602
|
.answer(url, what, authCode);
|
|
516
603
|
// If the answer data are valid:
|
|
@@ -526,12 +613,15 @@ const requestHandler = async (request, response) => {
|
|
|
526
613
|
}
|
|
527
614
|
// Otherwise, i.e. if it is a rejection:
|
|
528
615
|
else {
|
|
529
|
-
//
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
616
|
+
// Isolate this revision.
|
|
617
|
+
await recsLock(async () => {
|
|
618
|
+
// Get the recommendations.
|
|
619
|
+
const recs = await getRecs();
|
|
620
|
+
// Delete the rejected URL.
|
|
621
|
+
delete recs[url];
|
|
622
|
+
// Save the revised recommendations.
|
|
623
|
+
await fs.writeFile(path.join(jobsPath, 'recs.json'), getJSON(recs));
|
|
624
|
+
});
|
|
535
625
|
// Set a location header for a response.
|
|
536
626
|
response.setHeader('content-location', '/recActionForm.html');
|
|
537
627
|
// Get the answer data.
|
|
@@ -550,7 +640,7 @@ const requestHandler = async (request, response) => {
|
|
|
550
640
|
else if (pageName === 'reannotate.html') {
|
|
551
641
|
const {authCode} = postData;
|
|
552
642
|
// Set headers for a response.
|
|
553
|
-
setHeaders('text/html',
|
|
643
|
+
setHeaders('text/html', pathname, 'ultra');
|
|
554
644
|
// Get the answer data.
|
|
555
645
|
const answerData = await require(path.join(__dirname, 'reannotate', 'index'))
|
|
556
646
|
.answer(authCode);
|
|
@@ -569,7 +659,7 @@ const requestHandler = async (request, response) => {
|
|
|
569
659
|
else if (pageName === 'wcagRenew.html') {
|
|
570
660
|
const {authCode} = postData;
|
|
571
661
|
// Set headers for a response.
|
|
572
|
-
setHeaders('text/html',
|
|
662
|
+
setHeaders('text/html', pathname, 'low');
|
|
573
663
|
// Get the answer data.
|
|
574
664
|
const answerData = await require(path.join(__dirname, 'wcagRenew', 'index'))
|
|
575
665
|
.answer(authCode);
|
|
@@ -595,66 +685,8 @@ const requestHandler = async (request, response) => {
|
|
|
595
685
|
const service = segments[1];
|
|
596
686
|
// If the service is job assignment:
|
|
597
687
|
if (service === 'job') {
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
const jobNames = await getJobNames();
|
|
601
|
-
const claimedJobNames = jobNames.claimed;
|
|
602
|
-
// For each claimed job:
|
|
603
|
-
for (const jobName of claimedJobNames) {
|
|
604
|
-
const job = await getObject(path.join(jobsPath, 'claimed', jobName));
|
|
605
|
-
const {id, sources} = job;
|
|
606
|
-
const {agent} = sources;
|
|
607
|
-
// If its assignee is the agent:
|
|
608
|
-
if (agent === agentID) {
|
|
609
|
-
const messageEnd = `but has not completed job ${id}`;
|
|
610
|
-
// Report this.
|
|
611
|
-
await serveError({message: `${messageStart}${messageEnd}`}, response, false);
|
|
612
|
-
// Reclassify the job as failed.
|
|
613
|
-
await fs.rename(
|
|
614
|
-
path.join(claimedPath, jobName), path.join(failedPath, jobName)
|
|
615
|
-
);
|
|
616
|
-
clean = false;
|
|
617
|
-
// Stop checking claimed jobs.
|
|
618
|
-
break;
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
// If no aborted-job error was found for the agent:
|
|
622
|
-
if (clean) {
|
|
623
|
-
const queuedJobNames = jobNames.queue;
|
|
624
|
-
// If any jobs are queued:
|
|
625
|
-
if (queuedJobNames.length) {
|
|
626
|
-
const oldestJobName = queuedJobNames[0];
|
|
627
|
-
// Get the first one.
|
|
628
|
-
const firstJob = await getObject(path.join(queuePath, oldestJobName));
|
|
629
|
-
// Add the agent ID to the job.
|
|
630
|
-
firstJob.sources.agent = agentID;
|
|
631
|
-
console.log(
|
|
632
|
-
`Job ${firstJob.id} (${firstJob.target.what}) is being sent to the agent.`
|
|
633
|
-
);
|
|
634
|
-
// Assign the job to the agent.
|
|
635
|
-
response.writeHead(200, {
|
|
636
|
-
'content-type': 'application/json; charset=utf-8'
|
|
637
|
-
});
|
|
638
|
-
response.end(JSON.stringify(firstJob));
|
|
639
|
-
const messageEnd
|
|
640
|
-
= `and job ${firstJob.id} (${firstJob.target.what}) was assigned to the agent`;
|
|
641
|
-
console.log(`${messageStart}${messageEnd}`);
|
|
642
|
-
// Move the job from the queue to the claimed-jobs directory.
|
|
643
|
-
await fs.rename(
|
|
644
|
-
path.join(queuePath, oldestJobName), path.join(claimedPath, oldestJobName)
|
|
645
|
-
);
|
|
646
|
-
}
|
|
647
|
-
// Otherwise, i.e. if no jobs are queued:
|
|
648
|
-
else {
|
|
649
|
-
response.writeHead(200, {
|
|
650
|
-
'content-type': 'application/json; charset=utf-8'
|
|
651
|
-
});
|
|
652
|
-
// Send a no-jobs response to the agent.
|
|
653
|
-
response.end(JSON.stringify({}));
|
|
654
|
-
const messageEnd = 'but no job was in the queue';
|
|
655
|
-
console.log(`${messageStart}${messageEnd}`);
|
|
656
|
-
}
|
|
657
|
-
}
|
|
688
|
+
// Process the job request as a transaction.
|
|
689
|
+
await processJobRequest(request, response, agentID);
|
|
658
690
|
}
|
|
659
691
|
// Otherwise, if the service is report acquisition:
|
|
660
692
|
else if (service === 'report') {
|
|
@@ -664,91 +696,73 @@ const requestHandler = async (request, response) => {
|
|
|
664
696
|
const [timeStamp, jobID] = id?.split('-') ?? ['', ''];
|
|
665
697
|
// If the request is valid:
|
|
666
698
|
if (id && isTimeStamp(timeStamp) && isJobID(jobID) && what && url) {
|
|
667
|
-
// Acknowledge receipt.
|
|
668
|
-
response.setHeader('content-type', 'application/json; charset=utf-8');
|
|
669
|
-
response.end(JSON.stringify({status: 'ok'}));
|
|
670
|
-
console.log(`Testaro report ${id} was received from Testaro agent ${agentID}`);
|
|
671
699
|
const [timeStamp, jobID] = id.split('-');
|
|
672
700
|
// Save the report.
|
|
673
701
|
await fs.writeFile(getReportPath(timeStamp, jobID), getJSON(report));
|
|
674
|
-
//
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
url
|
|
678
|
-
};
|
|
679
|
-
// Save the log.
|
|
680
|
-
await fs.writeFile(getLogPath(timeStamp, jobID), getJSON(log));
|
|
681
|
-
// Annotate the report and mark it as annotated in the log.
|
|
702
|
+
// Update the data on the available reports.
|
|
703
|
+
await makeReportsExtract();
|
|
704
|
+
// Annotate the report.
|
|
682
705
|
await annotateReport(ruleIDs, timeStamp, jobID);
|
|
683
|
-
console.log(`Testaro report ${id} was annotated, saved, and
|
|
706
|
+
console.log(`Testaro report ${id} was annotated, saved, and indexed`);
|
|
684
707
|
// Check the monetary balances and send alerts if nearing exhaustion.
|
|
685
708
|
await checkBalancesForAlerts(report);
|
|
686
709
|
// Delete the job.
|
|
687
710
|
await fs.unlink(path.join(claimedPath, `${id}.json`));
|
|
688
711
|
console.log(`Completed job ${id} deleted`);
|
|
712
|
+
// Acknowledge receipt.
|
|
713
|
+
response.setHeader('content-type', 'application/json; charset=utf-8');
|
|
714
|
+
response.end(JSON.stringify({status: 'ok'}));
|
|
715
|
+
console.log(`Testaro report ${id} was received from Testaro agent ${agentID}`);
|
|
689
716
|
}
|
|
690
717
|
// Otherwise, i.e. if the request is invalid:
|
|
691
718
|
else {
|
|
692
|
-
await serveError({message: 'ERROR:
|
|
719
|
+
await serveError({message: 'ERROR: Request invalid'}, response, false);
|
|
693
720
|
}
|
|
694
721
|
}
|
|
695
|
-
// Otherwise, if the service is
|
|
722
|
+
// Otherwise, i.e. if the service is invalid:
|
|
696
723
|
else {
|
|
697
724
|
await serveError(
|
|
698
725
|
{message: 'ERROR: Invalid service request from Testaro agent'}, response, false
|
|
699
726
|
);
|
|
700
727
|
}
|
|
701
728
|
}
|
|
702
|
-
// Otherwise, if the first segment is the
|
|
703
|
-
else if (segments[0]
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
//
|
|
708
|
-
const
|
|
709
|
-
.response([description,
|
|
710
|
-
// Send
|
|
729
|
+
// Otherwise, if the first segment is not the ID of the Testaro agent:
|
|
730
|
+
else if (segments[0] !== testaroAgent) {
|
|
731
|
+
// If the service is to receive a test request:
|
|
732
|
+
if (segments[0] === 'requestTest') {
|
|
733
|
+
const {description, URL, reason} = postData;
|
|
734
|
+
// Get the response body.
|
|
735
|
+
const responseBody = await require(path.join(__dirname, 'api', 'requestTest'))
|
|
736
|
+
.response([description, URL, reason]);
|
|
737
|
+
// Send it.
|
|
711
738
|
setHeaders('application/json', null, 'ultra');
|
|
712
|
-
response.end(JSON.stringify(
|
|
739
|
+
response.end(JSON.stringify(responseBody));
|
|
713
740
|
}
|
|
714
|
-
// Otherwise,
|
|
715
|
-
else {
|
|
716
|
-
|
|
717
|
-
|
|
741
|
+
// Otherwise, if the service is to receive a retest request:
|
|
742
|
+
else if (segments[0] === 'requestRetest') {
|
|
743
|
+
const {reason} = postData;
|
|
744
|
+
// Get the response body.
|
|
745
|
+
const responseBody = await require(path.join(__dirname, 'api', 'requestRetest'))
|
|
746
|
+
.response(segments.slice(1).concat(reason));
|
|
747
|
+
// Send it.
|
|
748
|
+
setHeaders('application/json', null, 'ultra');
|
|
749
|
+
response.end(JSON.stringify(responseBody));
|
|
718
750
|
}
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
} = postData;
|
|
727
|
-
// If the payload is a valid test recommendation:
|
|
728
|
-
if (what && isURL(url) && why) {
|
|
729
|
-
let responseData;
|
|
730
|
-
// If a report on the page is already available:
|
|
731
|
-
if (await isReportAvailable(what, url)) {
|
|
732
|
-
// Report this.
|
|
733
|
-
responseData = {
|
|
734
|
-
status: 'warning',
|
|
735
|
-
message: 'A report on the page is already available'
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
// Otherwise, i.e. if no report on the page is available:
|
|
739
|
-
else {
|
|
740
|
-
// Process the recommendation and get the response data.
|
|
741
|
-
responseData = await require(path.join(__dirname, 'testRecForm', 'api'))
|
|
742
|
-
.response(what, url, why);
|
|
743
|
-
}
|
|
744
|
-
// Send the response data.
|
|
751
|
+
// Otherwise, if the service is to receive a feature request:
|
|
752
|
+
else if (segments[0] === 'requestFeature') {
|
|
753
|
+
const {request} = postData;
|
|
754
|
+
// Get the response body.
|
|
755
|
+
const responseBody = await require(path.join(__dirname, 'api', 'requestFeature'))
|
|
756
|
+
.response(segments.slice(1).concat(request));
|
|
757
|
+
// Send it.
|
|
745
758
|
setHeaders('application/json', null, 'ultra');
|
|
746
|
-
response.end(JSON.stringify(
|
|
759
|
+
response.end(JSON.stringify(responseBody));
|
|
747
760
|
}
|
|
748
|
-
// Otherwise, i.e. if
|
|
761
|
+
// Otherwise, i.e. if the service is invalid:
|
|
749
762
|
else {
|
|
750
|
-
|
|
751
|
-
|
|
763
|
+
await serveError(
|
|
764
|
+
{message: 'ERROR: Invalid request'}, response, false
|
|
765
|
+
);
|
|
752
766
|
}
|
|
753
767
|
}
|
|
754
768
|
// Otherwise, i.e. if the request is invalid:
|
|
@@ -790,7 +804,7 @@ const requestHandler = async (request, response) => {
|
|
|
790
804
|
|
|
791
805
|
const serve = async (protocolModule, options) => {
|
|
792
806
|
// Create any missing directories.
|
|
793
|
-
for (const path of [queuePath, claimedPath, failedPath,
|
|
807
|
+
for (const path of [queuePath, claimedPath, failedPath, hiddenReportsPath, reportsPath]) {
|
|
794
808
|
await fs.mkdir(path, {recursive: true});
|
|
795
809
|
}
|
|
796
810
|
const server = protocolModule === 'https'
|
package/issues/index.js
CHANGED
|
@@ -7,14 +7,12 @@
|
|
|
7
7
|
|
|
8
8
|
const {sendAlert} = require('../alerts');
|
|
9
9
|
const {
|
|
10
|
-
|
|
10
|
+
getLatestReportExtracts,
|
|
11
11
|
getReport,
|
|
12
12
|
getToolNamesString,
|
|
13
|
-
getLogs,
|
|
14
13
|
getWCAGLink,
|
|
15
14
|
getWeightName,
|
|
16
15
|
objectSort,
|
|
17
|
-
ruleIDs
|
|
18
16
|
} = require('../util');
|
|
19
17
|
const {issues} = require('testilo/procs/score/tic');
|
|
20
18
|
const fs = require('fs/promises');
|
|
@@ -23,20 +21,19 @@ const path = require('path');
|
|
|
23
21
|
// FUNCTIONS
|
|
24
22
|
|
|
25
23
|
// Gets summary data on the issues reported in a set of reports.
|
|
26
|
-
const getIssuesSummary = async
|
|
24
|
+
const getIssuesSummary = async () => {
|
|
27
25
|
// Initialize the summary.
|
|
28
|
-
const summary = {
|
|
26
|
+
const summary = {
|
|
27
|
+
totalCount: 0,
|
|
28
|
+
issues: []
|
|
29
|
+
};
|
|
29
30
|
// Initialize data for a summary.
|
|
30
31
|
const issuesData = {};
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if (! annotated) {
|
|
37
|
-
// Annotate it and mark it as annotated in the log.
|
|
38
|
-
await annotateReport(ruleIDs, timeStamp, jobID);
|
|
39
|
-
}
|
|
32
|
+
// Get extracts of the latest available report on each page.
|
|
33
|
+
const latestReportExtracts = await getLatestReportExtracts();
|
|
34
|
+
// For each of them:
|
|
35
|
+
for (const reportExtract of latestReportExtracts) {
|
|
36
|
+
const {timeStamp, jobID} = reportExtract;
|
|
40
37
|
// Get the corresponding report.
|
|
41
38
|
const report = await getReport(timeStamp, jobID);
|
|
42
39
|
const {acts = [], error} = report;
|
|
@@ -68,9 +65,6 @@ const getIssuesSummary = async logs => {
|
|
|
68
65
|
}
|
|
69
66
|
});
|
|
70
67
|
}
|
|
71
|
-
// Initialize the summary properties.
|
|
72
|
-
summary.totalCount = 0;
|
|
73
|
-
summary.issues = [];
|
|
74
68
|
// For each issue:
|
|
75
69
|
Object.entries(issuesData).forEach(([issueID, data]) => {
|
|
76
70
|
const {count, reporters} = data;
|
|
@@ -109,10 +103,8 @@ const getIssuesSummary = async logs => {
|
|
|
109
103
|
};
|
|
110
104
|
// Adds parameters to a query for the answer page.
|
|
111
105
|
const populateQuery = async query => {
|
|
112
|
-
// Get the logs of the latest reports on the tested targets.
|
|
113
|
-
const targetLogs = (await getLogs()).filter(log => ! log.superseded);
|
|
114
106
|
// Get summary data on the issues.
|
|
115
|
-
const issuesSummary = await getIssuesSummary(
|
|
107
|
+
const issuesSummary = await getIssuesSummary();
|
|
116
108
|
// If this failed:
|
|
117
109
|
if (issuesSummary.error) {
|
|
118
110
|
// Populate the query with the reason.
|
|
@@ -160,7 +152,7 @@ const populateQuery = async query => {
|
|
|
160
152
|
}
|
|
161
153
|
});
|
|
162
154
|
lines.push(`${margin}</ul>`);
|
|
163
|
-
if (!
|
|
155
|
+
if (!existsIssue) {
|
|
164
156
|
lines.push(`${margin}<p>No issues with this priority.</p>`);
|
|
165
157
|
}
|
|
166
158
|
});
|
package/jsconfig.json
ADDED
package/llms-full.txt
CHANGED
|
@@ -90,7 +90,7 @@ The implementation is currently in an alpha phase and offers these tools:
|
|
|
90
90
|
- `summarizeOneReport`
|
|
91
91
|
- method: `GET`
|
|
92
92
|
- purpose: summarize one report
|
|
93
|
-
- path: `/api/
|
|
93
|
+
- path: `/api/reportFacts/{timeStamp}/{jobID}`
|
|
94
94
|
- parameters
|
|
95
95
|
- `timeStamp`: initial segment of report identifier
|
|
96
96
|
- `jobID`: final segment of report identifier
|
|
@@ -156,7 +156,7 @@ More information about Kilotest features and internals:
|
|
|
156
156
|
|
|
157
157
|
### For language models and AI platforms
|
|
158
158
|
|
|
159
|
-
- [AI-
|
|
159
|
+
- [AI-TOOLS file](https://github.com/jrpool/kilotest/blob/main/docs/AI-TOOLS.md)
|
|
160
160
|
- [OpenAPI specification for Kilotest](https://kilotest.com/openapi.yaml)
|
|
161
161
|
- [Tool caller for testing](https://github.com/jrpool/kilotest/blob/main/researchAgent.js)
|
|
162
162
|
|