@jrpool/kilotest 59.0.2 → 59.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/api/orderNewTest.ts +1 -1
- package/api/orderRetest.ts +1 -1
- package/api/util.ts +1 -1
- package/capability.md +1 -1
- package/docs/AI-TOOLS.md +1 -1
- package/env.example +1 -1
- package/index.ts +5 -2
- package/lhm.plugin.json +25 -25
- package/llms-full.txt +1 -1
- package/llms.txt +1 -1
- package/mcp.test.ts +4 -4
- package/mcp.ts +31 -35
- package/openapi.yaml +2 -2
- package/package.json +1 -1
- package/scripts/generate-openapi.ts +1 -1
- package/server.json +2 -2
- package/test/dbFixture.ts +1 -0
- package/test/fixtures/buildFixtures.ts +1 -0
- package/util.test.ts +176 -2
- package/util.ts +74 -29
- package/web/home/index.html +4 -4
- package/web/listIssues/index.html +1 -1
- package/web/listIssues/index.ts +45 -24
- package/web/listIssues/listIssues.test.ts +3 -3
- package/web/listPages/listPages.test.ts +3 -3
- package/web/listTopIssues/listTopIssues.test.ts +37 -5
- package/web/reannotate/reannotate.test.ts +2 -2
- package/web/reannotateForm/reannotateForm.test.ts +37 -6
package/package.json
CHANGED
|
@@ -58,7 +58,7 @@ const generate = () => {
|
|
|
58
58
|
openapi: '3.1.0',
|
|
59
59
|
info: {
|
|
60
60
|
title: 'Kilotest API',
|
|
61
|
-
description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and standards
|
|
61
|
+
description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and web standards). Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access. With this API you can request that a page be tested or retested, discover available reports, incrementally retrieve facts from a report: from what reports are available, to what issues were reported in one report, to what elements violated one issue, to what diagnoses rule engines gave for one violation.',
|
|
62
62
|
version,
|
|
63
63
|
license: {
|
|
64
64
|
name: license,
|
package/server.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
3
|
"name": "io.github.jrpool/kilotest",
|
|
4
|
-
"description": "Ensemble testing of web pages for accessibility, usability, and standards
|
|
4
|
+
"description": "Ensemble testing of web pages for accessibility, usability, and web standards",
|
|
5
5
|
"repository": {
|
|
6
6
|
"url": "https://github.com/jrpool/kilotest",
|
|
7
7
|
"source": "github"
|
|
8
8
|
},
|
|
9
|
-
"version": "59.
|
|
9
|
+
"version": "59.1.0",
|
|
10
10
|
"remotes": [
|
|
11
11
|
{
|
|
12
12
|
"type": "streamable-http",
|
package/test/dbFixture.ts
CHANGED
|
@@ -24,6 +24,7 @@ fs.cpSync(path.join(import.meta.dirname, 'fixtures', 'db'), fixtureDBDir, {recur
|
|
|
24
24
|
for (const category of ['queue', 'claimed', 'failed']) {
|
|
25
25
|
fs.mkdirSync(path.join(fixtureDBDir, 'jobs', category), {recursive: true});
|
|
26
26
|
}
|
|
27
|
+
fs.mkdirSync(path.join(fixtureDBDir, 'unusableReports'), {recursive: true});
|
|
27
28
|
|
|
28
29
|
// Remove the copy when the process exits.
|
|
29
30
|
process.on('exit', () => fs.rmSync(fixtureDBDir, {recursive: true, force: true}));
|
|
@@ -114,6 +114,7 @@ const main = async () => {
|
|
|
114
114
|
await fs.mkdir(path.join(targetDir, 'jobs', 'claimed'), {recursive: true});
|
|
115
115
|
await fs.mkdir(path.join(targetDir, 'jobs', 'failed'), {recursive: true});
|
|
116
116
|
await fs.mkdir(path.join(targetDir, 'hiddenReports'), {recursive: true});
|
|
117
|
+
await fs.mkdir(path.join(targetDir, 'unusableReports'), {recursive: true});
|
|
117
118
|
|
|
118
119
|
// Fixture 1: mixedOutcomes. Two rule engines, four instances: two failed (same issue, same violator, two engines), one cantTell (excluded from counts), one failed (different issue, different violator). Two distinct issues (linkNoText weight 4, allCaps weight 1) and two distinct violator catalog indexes (0 and 1).
|
|
119
120
|
const mixedCatalog = {
|
package/util.test.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
awaitJob,
|
|
25
25
|
checkCommentDuplicate,
|
|
26
26
|
checkLength,
|
|
27
|
+
quarantineReport,
|
|
27
28
|
createLock,
|
|
28
29
|
dbPath,
|
|
29
30
|
describeMax,
|
|
@@ -61,7 +62,8 @@ import {
|
|
|
61
62
|
processTestRequest,
|
|
62
63
|
testRequestsLock,
|
|
63
64
|
testRequestsPath,
|
|
64
|
-
reportsPath
|
|
65
|
+
reportsPath,
|
|
66
|
+
unusableReportsPath
|
|
65
67
|
} from './util.ts';
|
|
66
68
|
|
|
67
69
|
// TESTS
|
|
@@ -142,7 +144,7 @@ test('dbPath honors DB_DIR when it is set', () => {
|
|
|
142
144
|
}
|
|
143
145
|
});
|
|
144
146
|
|
|
145
|
-
test('jobsPath, testRequestsPath, reportsPath, and
|
|
147
|
+
test('jobsPath, testRequestsPath, reportsPath, hiddenReportsPath, and unusableReportsPath derive from DB_DIR', () => {
|
|
146
148
|
const saved = process.env.DB_DIR;
|
|
147
149
|
process.env.DB_DIR = '/tmp/kilotest-fixtures';
|
|
148
150
|
try {
|
|
@@ -150,6 +152,7 @@ test('jobsPath, testRequestsPath, reportsPath, and hiddenReportsPath derive from
|
|
|
150
152
|
assert.equal(testRequestsPath(), path.join('/tmp/kilotest-fixtures', 'jobs', 'testRequests.json'));
|
|
151
153
|
assert.equal(reportsPath(), path.join('/tmp/kilotest-fixtures', 'reports'));
|
|
152
154
|
assert.equal(hiddenReportsPath(), path.join('/tmp/kilotest-fixtures', 'hiddenReports'));
|
|
155
|
+
assert.equal(unusableReportsPath(), path.join('/tmp/kilotest-fixtures', 'unusableReports'));
|
|
153
156
|
}
|
|
154
157
|
finally {
|
|
155
158
|
if (saved !== undefined) {
|
|
@@ -1445,6 +1448,177 @@ test('getReportExtract returns an error for a nonexistent report', async () => {
|
|
|
1445
1448
|
assert.ok(extract.error);
|
|
1446
1449
|
});
|
|
1447
1450
|
|
|
1451
|
+
test('getReportExtract returns an error for an unusable report', async () => {
|
|
1452
|
+
const os = await import('node:os');
|
|
1453
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-extract-bad-'));
|
|
1454
|
+
const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
|
|
1455
|
+
await fs.mkdir(tmpReportsDir, {recursive: true});
|
|
1456
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1457
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1458
|
+
try {
|
|
1459
|
+
// A report with a target and a completion time but no usable acts or catalog is
|
|
1460
|
+
// defective, so it must produce no extract rather than be exposed in listings.
|
|
1461
|
+
await fs.writeFile(
|
|
1462
|
+
path.join(tmpReportsDir, '260101T0000-bad.json'),
|
|
1463
|
+
JSON.stringify({
|
|
1464
|
+
target: {what: 'Defective Page', url: 'https://example.com/defective'},
|
|
1465
|
+
jobData: {endTime: '26-01-01T00:00'}
|
|
1466
|
+
})
|
|
1467
|
+
);
|
|
1468
|
+
const extract = await getReportExtract('260101T0000', 'bad') as any;
|
|
1469
|
+
assert.ok(extract.error);
|
|
1470
|
+
}
|
|
1471
|
+
finally {
|
|
1472
|
+
if (savedDbDir !== undefined) {
|
|
1473
|
+
process.env.DB_DIR = savedDbDir;
|
|
1474
|
+
}
|
|
1475
|
+
else {
|
|
1476
|
+
delete process.env.DB_DIR;
|
|
1477
|
+
}
|
|
1478
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1479
|
+
}
|
|
1480
|
+
});
|
|
1481
|
+
|
|
1482
|
+
test('quarantineReport ignores a missing report', async () => {
|
|
1483
|
+
const os = await import('node:os');
|
|
1484
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-missing-'));
|
|
1485
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1486
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1487
|
+
try {
|
|
1488
|
+
await quarantineReport('999999T9999', 'xxx', 'not usable');
|
|
1489
|
+
// Nothing was created or moved.
|
|
1490
|
+
assert.deepEqual(await fs.readdir(tmpRoot), []);
|
|
1491
|
+
}
|
|
1492
|
+
finally {
|
|
1493
|
+
if (savedDbDir !== undefined) {
|
|
1494
|
+
process.env.DB_DIR = savedDbDir;
|
|
1495
|
+
}
|
|
1496
|
+
else {
|
|
1497
|
+
delete process.env.DB_DIR;
|
|
1498
|
+
}
|
|
1499
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1500
|
+
}
|
|
1501
|
+
});
|
|
1502
|
+
|
|
1503
|
+
test('quarantineReport leaves a recently modified report in place', async () => {
|
|
1504
|
+
const os = await import('node:os');
|
|
1505
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-young-'));
|
|
1506
|
+
const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
|
|
1507
|
+
await fs.mkdir(tmpReportsDir, {recursive: true});
|
|
1508
|
+
const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
|
|
1509
|
+
await fs.writeFile(filePath, '{}');
|
|
1510
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1511
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1512
|
+
try {
|
|
1513
|
+
await quarantineReport('260101T0000', 'bad', 'not usable');
|
|
1514
|
+
// The fresh file may still be mid-write, so it was not moved.
|
|
1515
|
+
await fs.access(filePath);
|
|
1516
|
+
}
|
|
1517
|
+
finally {
|
|
1518
|
+
if (savedDbDir !== undefined) {
|
|
1519
|
+
process.env.DB_DIR = savedDbDir;
|
|
1520
|
+
}
|
|
1521
|
+
else {
|
|
1522
|
+
delete process.env.DB_DIR;
|
|
1523
|
+
}
|
|
1524
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1525
|
+
}
|
|
1526
|
+
});
|
|
1527
|
+
|
|
1528
|
+
test('quarantineReport quarantines a report old enough to be quarantined', async () => {
|
|
1529
|
+
const os = await import('node:os');
|
|
1530
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-old-'));
|
|
1531
|
+
const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
|
|
1532
|
+
await fs.mkdir(tmpReportsDir, {recursive: true});
|
|
1533
|
+
const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
|
|
1534
|
+
await fs.writeFile(filePath, '{}');
|
|
1535
|
+
// Age the file beyond the quarantine threshold.
|
|
1536
|
+
const longAgo = new Date(Date.now() - 120000);
|
|
1537
|
+
await fs.utimes(filePath, longAgo, longAgo);
|
|
1538
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1539
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1540
|
+
try {
|
|
1541
|
+
await quarantineReport('260101T0000', 'bad', 'not usable');
|
|
1542
|
+
// The file was moved to the unusable-reports directory.
|
|
1543
|
+
await fs.access(filePath).then(
|
|
1544
|
+
() => {throw new Error('Report should have been quarantined');},
|
|
1545
|
+
() => {}
|
|
1546
|
+
);
|
|
1547
|
+
const quarantined = await fs.readdir(path.join(tmpRoot, 'db', 'unusableReports'));
|
|
1548
|
+
assert.deepEqual(quarantined, ['260101T0000-bad.json']);
|
|
1549
|
+
}
|
|
1550
|
+
finally {
|
|
1551
|
+
if (savedDbDir !== undefined) {
|
|
1552
|
+
process.env.DB_DIR = savedDbDir;
|
|
1553
|
+
}
|
|
1554
|
+
else {
|
|
1555
|
+
delete process.env.DB_DIR;
|
|
1556
|
+
}
|
|
1557
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1558
|
+
}
|
|
1559
|
+
});
|
|
1560
|
+
|
|
1561
|
+
test('quarantineReport leaves the file and logs when the move fails', async () => {
|
|
1562
|
+
const os = await import('node:os');
|
|
1563
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-blocked-'));
|
|
1564
|
+
const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
|
|
1565
|
+
await fs.mkdir(tmpReportsDir, {recursive: true});
|
|
1566
|
+
const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
|
|
1567
|
+
await fs.writeFile(filePath, '{}');
|
|
1568
|
+
// Age the file beyond the quarantine threshold.
|
|
1569
|
+
const longAgo = new Date(Date.now() - 120000);
|
|
1570
|
+
await fs.utimes(filePath, longAgo, longAgo);
|
|
1571
|
+
// A directory already occupying the destination path makes the rename fail.
|
|
1572
|
+
await fs.mkdir(path.join(tmpRoot, 'db', 'unusableReports', '260101T0000-bad.json'), {recursive: true});
|
|
1573
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1574
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1575
|
+
try {
|
|
1576
|
+
await quarantineReport('260101T0000', 'bad', 'not usable');
|
|
1577
|
+
// The report stays where it was.
|
|
1578
|
+
await fs.access(filePath);
|
|
1579
|
+
}
|
|
1580
|
+
finally {
|
|
1581
|
+
if (savedDbDir !== undefined) {
|
|
1582
|
+
process.env.DB_DIR = savedDbDir;
|
|
1583
|
+
}
|
|
1584
|
+
else {
|
|
1585
|
+
delete process.env.DB_DIR;
|
|
1586
|
+
}
|
|
1587
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1588
|
+
}
|
|
1589
|
+
});
|
|
1590
|
+
|
|
1591
|
+
test('getReport quarantines a malformed report file old enough to be quarantined', async () => {
|
|
1592
|
+
const os = await import('node:os');
|
|
1593
|
+
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-getreport-bad-'));
|
|
1594
|
+
const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
|
|
1595
|
+
await fs.mkdir(tmpReportsDir, {recursive: true});
|
|
1596
|
+
const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
|
|
1597
|
+
await fs.writeFile(filePath, 'not JSON');
|
|
1598
|
+
// Age the file beyond the quarantine threshold.
|
|
1599
|
+
const longAgo = new Date(Date.now() - 120000);
|
|
1600
|
+
await fs.utimes(filePath, longAgo, longAgo);
|
|
1601
|
+
const savedDbDir = process.env.DB_DIR;
|
|
1602
|
+
process.env.DB_DIR = path.join(tmpRoot, 'db');
|
|
1603
|
+
try {
|
|
1604
|
+
const {getReport} = await import('./util.ts');
|
|
1605
|
+
const result = await getReport('260101T0000', 'bad') as any;
|
|
1606
|
+
assert.ok(result.error);
|
|
1607
|
+
// The malformed file was quarantined.
|
|
1608
|
+
const quarantined = await fs.readdir(path.join(tmpRoot, 'db', 'unusableReports'));
|
|
1609
|
+
assert.deepEqual(quarantined, ['260101T0000-bad.json']);
|
|
1610
|
+
}
|
|
1611
|
+
finally {
|
|
1612
|
+
if (savedDbDir !== undefined) {
|
|
1613
|
+
process.env.DB_DIR = savedDbDir;
|
|
1614
|
+
}
|
|
1615
|
+
else {
|
|
1616
|
+
delete process.env.DB_DIR;
|
|
1617
|
+
}
|
|
1618
|
+
await fs.rm(tmpRoot, {recursive: true, force: true});
|
|
1619
|
+
}
|
|
1620
|
+
});
|
|
1621
|
+
|
|
1448
1622
|
test('getReportExtracts returns extracts of all available reports', async () => {
|
|
1449
1623
|
const extracts = await getReportExtracts();
|
|
1450
1624
|
assert.ok(extracts.length >= 8);
|
package/util.ts
CHANGED
|
@@ -32,6 +32,9 @@ export const testRequestsPath = (): string => path.join(jobsPath(), 'testRequest
|
|
|
32
32
|
export const reportsPath = (): string => path.join(dbPath(), 'reports');
|
|
33
33
|
// Path of the hidden-reports directory.
|
|
34
34
|
export const hiddenReportsPath = (): string => path.join(dbPath(), 'hiddenReports');
|
|
35
|
+
// Path of the unusable-reports directory, where reports that fail the usability
|
|
36
|
+
// checks are quarantined for a manager to repair or delete.
|
|
37
|
+
export const unusableReportsPath = (): string => path.join(dbPath(), 'unusableReports');
|
|
35
38
|
// Path of the usage-metrics file.
|
|
36
39
|
export const metricsPath = (): string => path.join(dbPath(), 'metrics.json');
|
|
37
40
|
// IDs, names, and sponsors of Testaro rule engines.
|
|
@@ -692,7 +695,49 @@ export const isUsableReport = (report: unknown): report is UsableReport => {
|
|
|
692
695
|
&& !isNaN(getReportEndTime(r).getTime())
|
|
693
696
|
&& typeof r.catalog === 'object';
|
|
694
697
|
};
|
|
695
|
-
//
|
|
698
|
+
// Minimum age, in milliseconds, of the last modification of a report file for the
|
|
699
|
+
// file to be quarantined. A more recently modified file may still be being written
|
|
700
|
+
// by a worker or a reannotation, so quarantining it could mistake an incomplete write
|
|
701
|
+
// for a defect. Read at call time so tests can shrink it via REPORT_QUARANTINE_MIN_AGE_MS;
|
|
702
|
+
// a value of 0 disables the age check.
|
|
703
|
+
const quarantineMinAgeMs = () => getEnvMax('REPORT_QUARANTINE_MIN_AGE_MS', 60000);
|
|
704
|
+
// Moves a report file from the reports directory to the unusable-reports directory
|
|
705
|
+
// and alerts a manager, if the file exists and is old enough that it can no longer
|
|
706
|
+
// plausibly be mid-write. Called by getReport whenever a report proves defective, so
|
|
707
|
+
// that a defective report is quarantined rather than merely hidden from whichever
|
|
708
|
+
// listing encountered it. Never throws: a failure here must not displace the error
|
|
709
|
+
// that getReport is already reporting.
|
|
710
|
+
export const quarantineReport = async (timeStamp: string, jobID: string, defect: string): Promise<void> => {
|
|
711
|
+
const jobName = `${timeStamp}-${jobID}`;
|
|
712
|
+
// Get the modification time of the report file. A missing file needs no move.
|
|
713
|
+
const stat = await fs.stat(getReportPath(timeStamp, jobID)).catch(() => null);
|
|
714
|
+
if (!stat) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
// Leave a recently modified file alone, since it may still be being written.
|
|
718
|
+
if (Date.now() - stat.mtimeMs < quarantineMinAgeMs()) {
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
try {
|
|
722
|
+
await fs.mkdir(unusableReportsPath(), {recursive: true});
|
|
723
|
+
await fs.rename(
|
|
724
|
+
getReportPath(timeStamp, jobID), path.join(unusableReportsPath(), `${jobName}.json`)
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
catch (error: unknown) {
|
|
728
|
+
// The report stays excluded regardless, so a failed move is only logged.
|
|
729
|
+
console.error(`ERROR: Failed to quarantine report ${jobName} (${errorMessage(error)})`);
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
// Alert a manager, who can repair the report in place or delete it.
|
|
733
|
+
await sendAlert(
|
|
734
|
+
'Kilotest: unusable report',
|
|
735
|
+
`Report ${jobName} is ${defect}. It was moved to the unusableReports directory, where a manager can repair or delete it.`
|
|
736
|
+
);
|
|
737
|
+
};
|
|
738
|
+
// Returns a report, or an error object if it is missing, unreadable, unparsable, or
|
|
739
|
+
// unusable. A report that exists but cannot be read or is unusable is defective, so
|
|
740
|
+
// it is quarantined rather than merely rejected by whichever read encountered it.
|
|
696
741
|
export const getReport = async (timeStamp: string, jobID: string): Promise<UsableReport | {error: string}> => {
|
|
697
742
|
try {
|
|
698
743
|
const reportJSON = await fs.readFile(getReportPath(timeStamp, jobID), 'utf8');
|
|
@@ -702,11 +747,15 @@ export const getReport = async (timeStamp: string, jobID: string): Promise<Usabl
|
|
|
702
747
|
// Return it.
|
|
703
748
|
return report;
|
|
704
749
|
}
|
|
705
|
-
// Otherwise, i.e. if it is unusable, return this.
|
|
706
|
-
return {error: `Report ${timeStamp}-${jobID} is not usable`};
|
|
707
750
|
} catch (error: unknown) {
|
|
751
|
+
// The file exists but cannot be read or parsed, or is already missing.
|
|
752
|
+
// quarantineReport ignores a missing file, so no existence check is needed here.
|
|
753
|
+
await quarantineReport(timeStamp, jobID, 'unreadable or not JSON');
|
|
708
754
|
return {error: `Report ${timeStamp}-${jobID} is missing, unreadable, or not JSON (${errorMessage(error)})`};
|
|
709
755
|
}
|
|
756
|
+
// The report was read but is not usable, so quarantine it.
|
|
757
|
+
await quarantineReport(timeStamp, jobID, 'not usable');
|
|
758
|
+
return {error: `Report ${timeStamp}-${jobID} is not usable`};
|
|
710
759
|
};
|
|
711
760
|
// Returns whether a getReport result is an error (rather than a usable report).
|
|
712
761
|
// A type predicate is used instead of `if (report.error)` because UsableReport
|
|
@@ -773,30 +822,27 @@ export const getReportEndTime = (report: {jobData?: {endTime?: string}}): Date =
|
|
|
773
822
|
`20${report.jobData?.endTime}Z`
|
|
774
823
|
);
|
|
775
824
|
|
|
776
|
-
// Returns an extract of an available report, or an error object if
|
|
825
|
+
// Returns an extract of an available report, or an error object if the report cannot
|
|
826
|
+
// be used. An extract exists to expose a report in listings, so it shares getReport's
|
|
827
|
+
// usability bar: a report too defective to expose must produce no extract.
|
|
777
828
|
export const getReportExtract = async (timeStamp: string, jobID: string): Promise<ReportExtract | {error: string}> => {
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
const report = JSON.parse(reportJSON);
|
|
784
|
-
const {target, jobData} = report;
|
|
785
|
-
const {what: description, url} = target;
|
|
786
|
-
// Return an extract of it.
|
|
787
|
-
return {
|
|
788
|
-
timeStamp,
|
|
789
|
-
jobID,
|
|
790
|
-
description,
|
|
791
|
-
url,
|
|
792
|
-
reportTime: getReportEndTime({jobData}).toISOString()
|
|
793
|
-
};
|
|
794
|
-
}
|
|
795
|
-
catch {
|
|
796
|
-
return {
|
|
797
|
-
error: `No report ${timeStamp}-${jobID} is available`
|
|
798
|
-
};
|
|
829
|
+
const report = await getReport(timeStamp, jobID);
|
|
830
|
+
// If the report is not usable:
|
|
831
|
+
if (isReportError(report)) {
|
|
832
|
+
// Return the reason.
|
|
833
|
+
return report;
|
|
799
834
|
}
|
|
835
|
+
const {target, jobData} = report;
|
|
836
|
+
const {what: description, url} = target;
|
|
837
|
+
// Return an extract of it. Usability has already guaranteed these properties,
|
|
838
|
+
// including a valid completion time.
|
|
839
|
+
return {
|
|
840
|
+
timeStamp,
|
|
841
|
+
jobID,
|
|
842
|
+
description,
|
|
843
|
+
url,
|
|
844
|
+
reportTime: getReportEndTime({jobData}).toISOString()
|
|
845
|
+
};
|
|
800
846
|
};
|
|
801
847
|
// Cache of report extracts, keyed by report file path. Each entry records the modification
|
|
802
848
|
// time and size of the file when it was extracted, so an entry is reused only while the file
|
|
@@ -804,7 +850,7 @@ export const getReportExtract = async (timeStamp: string, jobID: string): Promis
|
|
|
804
850
|
// written, hidden, or deleted by any process, so the cache is validated against the files
|
|
805
851
|
// before every use rather than invalidated by the code that changes them.
|
|
806
852
|
const reportExtractCache = new Map<string, {mtimeMs: number, size: number, extract: ReportExtract}>();
|
|
807
|
-
// Returns extracts of all available reports.
|
|
853
|
+
// Returns extracts of all available reports, ordered by page description and completion time.
|
|
808
854
|
export const getReportExtracts = async (onlyLatest: boolean = false): Promise<ReportExtract[]> => {
|
|
809
855
|
const reportsDir = reportsPath();
|
|
810
856
|
// Get the names and paths of the available report files.
|
|
@@ -866,8 +912,7 @@ export const getReportExtracts = async (onlyLatest: boolean = false): Promise<Re
|
|
|
866
912
|
// Return the array, excluding extracts of superseded reports if so specified.
|
|
867
913
|
return onlyLatest ? extracts.filter(extract => !extract.superseded) : extracts;
|
|
868
914
|
};
|
|
869
|
-
// Returns extracts of all available reports about
|
|
870
|
-
// to latest.
|
|
915
|
+
// Returns extracts of all available reports about a page, from oldest to latest.
|
|
871
916
|
export const getPageReportExtracts = async (description: string): Promise<ReportExtract[]> => {
|
|
872
917
|
const reportExtracts = await getReportExtracts();
|
|
873
918
|
return reportExtracts.filter(extract => extract.description === description);
|
|
@@ -1160,7 +1205,7 @@ export const orderJob = (
|
|
|
1160
1205
|
// waits before giving up. Both are read at call time (not cached), via environment
|
|
1161
1206
|
// variables, so tests can shrink them well below the real ~2-to-4-minute wait a
|
|
1162
1207
|
// production caller experiences; AWAIT_JOB_TIMEOUT_MS deliberately exceeds the
|
|
1163
|
-
// documented "
|
|
1208
|
+
// documented "2 to 4 minutes" a caller is told to expect, leaving margin
|
|
1164
1209
|
// so a job that is genuinely still running is not mistaken for one that never will
|
|
1165
1210
|
// finish.
|
|
1166
1211
|
const awaitJobPollMs = () => getEnvMax('AWAIT_JOB_POLL_MS', 2000);
|
package/web/home/index.html
CHANGED
|
@@ -16,15 +16,15 @@
|
|
|
16
16
|
"sameAs": "https://github.com/jrpool/kilotest",
|
|
17
17
|
"datePublished": "2025-10-14",
|
|
18
18
|
"dateModified": "2026-08-26",
|
|
19
|
-
"applicationCategory": "Front-end quality (i.e.
|
|
19
|
+
"applicationCategory": "Front-end quality (i.e. accessibility, usability, and web standards) testing and reporting service",
|
|
20
20
|
"isAccessibleForFree": true,
|
|
21
|
-
"description": "Testing of web pages for front-end quality (i.e.
|
|
21
|
+
"description": "Testing of web pages for front-end quality (i.e. accessibility, usability, and web standards) with an ensemble of 12 rule engines and provision of results at selectable levels of detail.",
|
|
22
22
|
"softwareHelp": {
|
|
23
23
|
"@type": "CreativeWork",
|
|
24
24
|
"url": "https://kilotest.com/tutorialWeb.html"
|
|
25
25
|
},
|
|
26
26
|
"featureList": [
|
|
27
|
-
"Automated testing of front-end quality (i.e.
|
|
27
|
+
"Automated testing of front-end quality (i.e. accessibility, usability, and web standards)",
|
|
28
28
|
"WCAG 2.2 conformance evaluation",
|
|
29
29
|
"HTML5 conformance evaluation",
|
|
30
30
|
"CSS3 conformance evaluation",
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
<details>
|
|
63
63
|
<summary>More than a thousand? Really?</summary>
|
|
64
64
|
<h2>Ensemble testing</h2>
|
|
65
|
-
<p>Kilotest does <strong>ensemble testing</strong> of web pages for <a href="https://www.w3.org/mission/accessibility/">accessibility</a>, usability, and standards
|
|
65
|
+
<p>Kilotest does <strong>ensemble testing</strong> of web pages for <a href="https://www.w3.org/mission/accessibility/">accessibility</a>, usability, and web standards.</p>
|
|
66
66
|
<p>Efficient testing <strong>first</strong> leverages <strong>automated rule engines</strong> and corrects the defects that they discover, <strong>then</strong> employs human testers.</p>
|
|
67
67
|
<p>Choosing among the <a href="https://www.w3.org/WAI/test-evaluate/tools/list/">many automated rule engines</a> is hard, because they <a href="https://arxiv.org/pdf/2304.07591">vary in what and how they test</a>.</p>
|
|
68
68
|
<p><strong>Ensemble testing</strong>, instead of choosing only one, combines <strong>multiple rule engines</strong>. It can discover more issues than single-rule-engine testing, and the rule engines can check one another’s quality.</p>
|
package/web/listIssues/index.ts
CHANGED
|
@@ -123,46 +123,62 @@ const getIssuesData = (report: UsableReport): ResultsSummary & Record<string, an
|
|
|
123
123
|
// Return the data.
|
|
124
124
|
return final;
|
|
125
125
|
};
|
|
126
|
-
// Returns
|
|
127
|
-
const
|
|
128
|
-
timeStamp: string, jobID: string, description: string, margin: string
|
|
129
|
-
): Promise<string[]> => {
|
|
126
|
+
// Returns data and HTML lines about the history of the reports about the page of a report.
|
|
127
|
+
const getHistory = async (
|
|
128
|
+
timeStamp: string, jobID: string, description: string, margin: string = ''
|
|
129
|
+
): Promise<{isLatest: boolean, lines: string[]}> => {
|
|
130
130
|
// Get extracts of all reports about the page, from oldest to latest.
|
|
131
131
|
const pageExtracts = await getPageReportExtracts(description);
|
|
132
|
-
|
|
133
|
-
//
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
132
|
+
// The report's own extract may be absent if the report was quarantined between the
|
|
133
|
+
// report read and the extract listing, so an empty list means "not latest".
|
|
134
|
+
const lastExtract = pageExtracts[pageExtracts.length - 1];
|
|
135
|
+
const isLatest = lastExtract !== undefined
|
|
136
|
+
&& lastExtract.timeStamp === timeStamp && lastExtract.jobID === jobID;
|
|
137
|
+
// Initialize the history.
|
|
138
|
+
const history = {
|
|
139
|
+
isLatest,
|
|
140
|
+
lines: [] as string[]
|
|
141
|
+
};
|
|
142
|
+
// For each report about the page, from oldest to latest:
|
|
137
143
|
pageExtracts.forEach((extract, index) => {
|
|
144
|
+
// Get whether it is this report.
|
|
138
145
|
const isThis = extract.timeStamp === timeStamp && extract.jobID === jobID;
|
|
139
|
-
isLatest ||= isThis && index === lastIndex;
|
|
140
146
|
// Get the tags that identify its position and whether it is this report.
|
|
141
147
|
const tags = [
|
|
142
148
|
index === 0 ? 'oldest' : '',
|
|
143
|
-
index ===
|
|
149
|
+
index === pageExtracts.length - 1 ? 'latest' : '',
|
|
144
150
|
isThis ? 'this report' : ''
|
|
145
151
|
]
|
|
146
152
|
.filter(tag => tag);
|
|
153
|
+
// Get a string containing the applicable tags.
|
|
147
154
|
const tagString = tags.length ? ` (${tags.join('; ')})` : '';
|
|
148
155
|
const label = `${getAgoString(new Date(extract.reportTime))} ago${tagString}`;
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
156
|
+
let content;
|
|
157
|
+
if (isThis) {
|
|
158
|
+
content = label;
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
const href = `/listIssues.html/${extract.timeStamp}/${extract.jobID}`;
|
|
162
|
+
content = `<a href="${href}">${label}</a>`;
|
|
163
|
+
}
|
|
164
|
+
// Add a list item describing the report, linked if not this report, to the history.
|
|
165
|
+
history.lines.push(`${margin}<li>${content}</li>`);
|
|
153
166
|
});
|
|
154
167
|
// Initialize the lines with a list of the reports, or a statement that there is only one.
|
|
155
|
-
const lines =
|
|
156
|
-
? [`${margin}<ol>`, ...
|
|
157
|
-
: [`${margin}<p>This is the only report about the page.</p>`];
|
|
158
|
-
// If
|
|
168
|
+
const lines = history.lines.length > 1
|
|
169
|
+
? [`${margin}<ol>`, ...history.lines, `${margin}</ol>`]
|
|
170
|
+
: [`${margin}<p>This is the only available report about the page.</p>`];
|
|
171
|
+
// If this report is the latest about the page:
|
|
159
172
|
if (isLatest) {
|
|
160
173
|
// Add a link to the form for requesting a retest to the lines.
|
|
161
|
-
|
|
162
|
-
|
|
174
|
+
lines.push(
|
|
175
|
+
`${margin}<p><a href="/requestRetestForm.html/${timeStamp}/${jobID}">Request a retest</a></p>`
|
|
176
|
+
);
|
|
163
177
|
}
|
|
164
|
-
//
|
|
165
|
-
|
|
178
|
+
// Replace the bare list items with the assembled section lines.
|
|
179
|
+
history.lines = lines;
|
|
180
|
+
// Return the history.
|
|
181
|
+
return history;
|
|
166
182
|
};
|
|
167
183
|
// Adds parameters to a query for the answer page.
|
|
168
184
|
const populateQuery = async (timeStamp: string, jobID: string, query: Record<string, any>) => {
|
|
@@ -191,10 +207,15 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record<str
|
|
|
191
207
|
query.target = description;
|
|
192
208
|
query.testInfo = testInfo;
|
|
193
209
|
const margin = ' '.repeat(6);
|
|
210
|
+
const history = await getHistory(timeStamp, jobID, description, ' '.repeat(8));
|
|
211
|
+
// Add a warning if the report is not the latest about the page.
|
|
212
|
+
query.oldWarn = history.isLatest
|
|
213
|
+
? ''
|
|
214
|
+
: '<p><strong>Warning: This report is not the latest about the page.</strong> See history below.</p>';
|
|
194
215
|
// Add the page-facts list to the query.
|
|
195
216
|
query.pageFacts = getPageFactsLines(pageInfo, issuesData, margin).join('\n');
|
|
196
217
|
// Add the history of reports about the page to the query.
|
|
197
|
-
query.history =
|
|
218
|
+
query.history = history.lines.join('\n');
|
|
198
219
|
const {issues} = issuesData;
|
|
199
220
|
// Add a summary of the issues to the query.
|
|
200
221
|
query.highestCount = issues[4].length;
|
|
@@ -231,7 +231,7 @@ test('listIssues History of an older report lists all reports, with this one unl
|
|
|
231
231
|
assert.equal(items[0]!.href, null);
|
|
232
232
|
assert.match(items[1]!.text, /^\d+ days? ago \(latest\)$/);
|
|
233
233
|
assert.equal(items[1]!.href, '/listIssues.html/260202T0000/new');
|
|
234
|
-
assert.ok(!result.answerPage.includes('
|
|
234
|
+
assert.ok(!result.answerPage.includes('Request a retest'));
|
|
235
235
|
});
|
|
236
236
|
|
|
237
237
|
test('listIssues History of the latest report ends with the retest question', async () => {
|
|
@@ -244,7 +244,7 @@ test('listIssues History of the latest report ends with the retest question', as
|
|
|
244
244
|
assert.match(items[1]!.text, /^\d+ days? ago \(latest; this report\)$/);
|
|
245
245
|
assert.equal(items[1]!.href, null);
|
|
246
246
|
const retestLink = history.querySelector('ol + p > a');
|
|
247
|
-
assert.equal(retestLink?.text, '
|
|
247
|
+
assert.equal(retestLink?.text, 'Request a retest');
|
|
248
248
|
assert.equal(retestLink?.getAttribute('href'), '/requestRetestForm.html/260202T0000/new');
|
|
249
249
|
});
|
|
250
250
|
|
|
@@ -253,7 +253,7 @@ test('listIssues History of the only report about a page says so and asks about
|
|
|
253
253
|
const history = getHistory(result.answerPage);
|
|
254
254
|
assert.equal(getHistoryItems(history).length, 0);
|
|
255
255
|
const paragraphs = history.querySelectorAll('p');
|
|
256
|
-
assert.equal(paragraphs[0]?.text, 'This is the only report about the page.');
|
|
256
|
+
assert.equal(paragraphs[0]?.text, 'This is the only available report about the page.');
|
|
257
257
|
assert.equal(paragraphs[1]?.querySelector('a')?.getAttribute('href'), '/requestRetestForm.html/260101T0001/ct');
|
|
258
258
|
});
|
|
259
259
|
|
|
@@ -149,12 +149,12 @@ test('listPages shows queued and claimed jobs when they exist', async () => {
|
|
|
149
149
|
}
|
|
150
150
|
});
|
|
151
151
|
|
|
152
|
-
test('listPages
|
|
152
|
+
test('listPages omits a page whose report is not usable', {timeout: 500}, async () => {
|
|
153
153
|
const dbDir = (await import('../../test/dbFixture.ts')).fixtureDBDir;
|
|
154
154
|
const reportsDir = path.join(dbDir, 'reports');
|
|
155
155
|
const invalidReportPath = path.join(reportsDir, '260101T9999-bad.json');
|
|
156
156
|
try {
|
|
157
|
-
//
|
|
157
|
+
// Not usable by isUsableReport (no acts or catalog), so it must not be exposed.
|
|
158
158
|
const invalidReport = {
|
|
159
159
|
target: {what: 'Invalid Report', url: 'https://example.com/invalid'},
|
|
160
160
|
jobData: {endTime: '26-01-01T00:10'}
|
|
@@ -163,7 +163,7 @@ test('listPages lists a page whose report is not usable', {timeout: 500}, async
|
|
|
163
163
|
const result: any = await answer();
|
|
164
164
|
assert.equal(result.status, 'ok');
|
|
165
165
|
const links = getPageLinks(result.answerPage);
|
|
166
|
-
assert.ok(links.some(a => a.text === 'Invalid Report'));
|
|
166
|
+
assert.ok(!links.some(a => a.text === 'Invalid Report'));
|
|
167
167
|
}
|
|
168
168
|
finally {
|
|
169
169
|
await fs.unlink(invalidReportPath).catch(() => {});
|