@jrpool/kilotest 59.0.2 → 59.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jrpool/kilotest",
3
- "version": "59.0.2",
3
+ "version": "59.1.0",
4
4
  "description": "Ensemble testing for front-end web quality",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -58,7 +58,7 @@ const generate = () => {
58
58
  openapi: '3.1.0',
59
59
  info: {
60
60
  title: 'Kilotest API',
61
- description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and standards conformity). Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access. With this API you can request that a page be tested or retested, discover available reports, incrementally retrieve facts from a report: from what reports are available, to what issues were reported in one report, to what elements violated one issue, to what diagnoses rule engines gave for one violation.',
61
+ description: 'Kilotest is an ensemble testing and reporting service for front-end web quality. It uses an ensemble of 12 rule engines to test web pages for front-end quality (accessibility, usability, and web standards). Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access. With this API you can request that a page be tested or retested, discover available reports, incrementally retrieve facts from a report: from what reports are available, to what issues were reported in one report, to what elements violated one issue, to what diagnoses rule engines gave for one violation.',
62
62
  version,
63
63
  license: {
64
64
  name: license,
package/server.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
3
3
  "name": "io.github.jrpool/kilotest",
4
- "description": "Ensemble testing of web pages for accessibility, usability, and standards conformity",
4
+ "description": "Ensemble testing of web pages for accessibility, usability, and web standards",
5
5
  "repository": {
6
6
  "url": "https://github.com/jrpool/kilotest",
7
7
  "source": "github"
8
8
  },
9
- "version": "59.0.0",
9
+ "version": "59.1.0",
10
10
  "remotes": [
11
11
  {
12
12
  "type": "streamable-http",
package/test/dbFixture.ts CHANGED
@@ -24,6 +24,7 @@ fs.cpSync(path.join(import.meta.dirname, 'fixtures', 'db'), fixtureDBDir, {recur
24
24
  for (const category of ['queue', 'claimed', 'failed']) {
25
25
  fs.mkdirSync(path.join(fixtureDBDir, 'jobs', category), {recursive: true});
26
26
  }
27
+ fs.mkdirSync(path.join(fixtureDBDir, 'unusableReports'), {recursive: true});
27
28
 
28
29
  // Remove the copy when the process exits.
29
30
  process.on('exit', () => fs.rmSync(fixtureDBDir, {recursive: true, force: true}));
@@ -114,6 +114,7 @@ const main = async () => {
114
114
  await fs.mkdir(path.join(targetDir, 'jobs', 'claimed'), {recursive: true});
115
115
  await fs.mkdir(path.join(targetDir, 'jobs', 'failed'), {recursive: true});
116
116
  await fs.mkdir(path.join(targetDir, 'hiddenReports'), {recursive: true});
117
+ await fs.mkdir(path.join(targetDir, 'unusableReports'), {recursive: true});
117
118
 
118
119
  // Fixture 1: mixedOutcomes. Two rule engines, four instances: two failed (same issue, same violator, two engines), one cantTell (excluded from counts), one failed (different issue, different violator). Two distinct issues (linkNoText weight 4, allCaps weight 1) and two distinct violator catalog indexes (0 and 1).
119
120
  const mixedCatalog = {
package/util.test.ts CHANGED
@@ -24,6 +24,7 @@ import {
24
24
  awaitJob,
25
25
  checkCommentDuplicate,
26
26
  checkLength,
27
+ quarantineReport,
27
28
  createLock,
28
29
  dbPath,
29
30
  describeMax,
@@ -61,7 +62,8 @@ import {
61
62
  processTestRequest,
62
63
  testRequestsLock,
63
64
  testRequestsPath,
64
- reportsPath
65
+ reportsPath,
66
+ unusableReportsPath
65
67
  } from './util.ts';
66
68
 
67
69
  // TESTS
@@ -142,7 +144,7 @@ test('dbPath honors DB_DIR when it is set', () => {
142
144
  }
143
145
  });
144
146
 
145
- test('jobsPath, testRequestsPath, reportsPath, and hiddenReportsPath derive from DB_DIR', () => {
147
+ test('jobsPath, testRequestsPath, reportsPath, hiddenReportsPath, and unusableReportsPath derive from DB_DIR', () => {
146
148
  const saved = process.env.DB_DIR;
147
149
  process.env.DB_DIR = '/tmp/kilotest-fixtures';
148
150
  try {
@@ -150,6 +152,7 @@ test('jobsPath, testRequestsPath, reportsPath, and hiddenReportsPath derive from
150
152
  assert.equal(testRequestsPath(), path.join('/tmp/kilotest-fixtures', 'jobs', 'testRequests.json'));
151
153
  assert.equal(reportsPath(), path.join('/tmp/kilotest-fixtures', 'reports'));
152
154
  assert.equal(hiddenReportsPath(), path.join('/tmp/kilotest-fixtures', 'hiddenReports'));
155
+ assert.equal(unusableReportsPath(), path.join('/tmp/kilotest-fixtures', 'unusableReports'));
153
156
  }
154
157
  finally {
155
158
  if (saved !== undefined) {
@@ -1445,6 +1448,177 @@ test('getReportExtract returns an error for a nonexistent report', async () => {
1445
1448
  assert.ok(extract.error);
1446
1449
  });
1447
1450
 
1451
+ test('getReportExtract returns an error for an unusable report', async () => {
1452
+ const os = await import('node:os');
1453
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-extract-bad-'));
1454
+ const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
1455
+ await fs.mkdir(tmpReportsDir, {recursive: true});
1456
+ const savedDbDir = process.env.DB_DIR;
1457
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1458
+ try {
1459
+ // A report with a target and a completion time but no usable acts or catalog is
1460
+ // defective, so it must produce no extract rather than be exposed in listings.
1461
+ await fs.writeFile(
1462
+ path.join(tmpReportsDir, '260101T0000-bad.json'),
1463
+ JSON.stringify({
1464
+ target: {what: 'Defective Page', url: 'https://example.com/defective'},
1465
+ jobData: {endTime: '26-01-01T00:00'}
1466
+ })
1467
+ );
1468
+ const extract = await getReportExtract('260101T0000', 'bad') as any;
1469
+ assert.ok(extract.error);
1470
+ }
1471
+ finally {
1472
+ if (savedDbDir !== undefined) {
1473
+ process.env.DB_DIR = savedDbDir;
1474
+ }
1475
+ else {
1476
+ delete process.env.DB_DIR;
1477
+ }
1478
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1479
+ }
1480
+ });
1481
+
1482
+ test('quarantineReport ignores a missing report', async () => {
1483
+ const os = await import('node:os');
1484
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-missing-'));
1485
+ const savedDbDir = process.env.DB_DIR;
1486
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1487
+ try {
1488
+ await quarantineReport('999999T9999', 'xxx', 'not usable');
1489
+ // Nothing was created or moved.
1490
+ assert.deepEqual(await fs.readdir(tmpRoot), []);
1491
+ }
1492
+ finally {
1493
+ if (savedDbDir !== undefined) {
1494
+ process.env.DB_DIR = savedDbDir;
1495
+ }
1496
+ else {
1497
+ delete process.env.DB_DIR;
1498
+ }
1499
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1500
+ }
1501
+ });
1502
+
1503
+ test('quarantineReport leaves a recently modified report in place', async () => {
1504
+ const os = await import('node:os');
1505
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-young-'));
1506
+ const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
1507
+ await fs.mkdir(tmpReportsDir, {recursive: true});
1508
+ const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
1509
+ await fs.writeFile(filePath, '{}');
1510
+ const savedDbDir = process.env.DB_DIR;
1511
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1512
+ try {
1513
+ await quarantineReport('260101T0000', 'bad', 'not usable');
1514
+ // The fresh file may still be mid-write, so it was not moved.
1515
+ await fs.access(filePath);
1516
+ }
1517
+ finally {
1518
+ if (savedDbDir !== undefined) {
1519
+ process.env.DB_DIR = savedDbDir;
1520
+ }
1521
+ else {
1522
+ delete process.env.DB_DIR;
1523
+ }
1524
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1525
+ }
1526
+ });
1527
+
1528
+ test('quarantineReport quarantines a report old enough to be quarantined', async () => {
1529
+ const os = await import('node:os');
1530
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-old-'));
1531
+ const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
1532
+ await fs.mkdir(tmpReportsDir, {recursive: true});
1533
+ const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
1534
+ await fs.writeFile(filePath, '{}');
1535
+ // Age the file beyond the quarantine threshold.
1536
+ const longAgo = new Date(Date.now() - 120000);
1537
+ await fs.utimes(filePath, longAgo, longAgo);
1538
+ const savedDbDir = process.env.DB_DIR;
1539
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1540
+ try {
1541
+ await quarantineReport('260101T0000', 'bad', 'not usable');
1542
+ // The file was moved to the unusable-reports directory.
1543
+ await fs.access(filePath).then(
1544
+ () => {throw new Error('Report should have been quarantined');},
1545
+ () => {}
1546
+ );
1547
+ const quarantined = await fs.readdir(path.join(tmpRoot, 'db', 'unusableReports'));
1548
+ assert.deepEqual(quarantined, ['260101T0000-bad.json']);
1549
+ }
1550
+ finally {
1551
+ if (savedDbDir !== undefined) {
1552
+ process.env.DB_DIR = savedDbDir;
1553
+ }
1554
+ else {
1555
+ delete process.env.DB_DIR;
1556
+ }
1557
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1558
+ }
1559
+ });
1560
+
1561
+ test('quarantineReport leaves the file and logs when the move fails', async () => {
1562
+ const os = await import('node:os');
1563
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-quarantine-blocked-'));
1564
+ const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
1565
+ await fs.mkdir(tmpReportsDir, {recursive: true});
1566
+ const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
1567
+ await fs.writeFile(filePath, '{}');
1568
+ // Age the file beyond the quarantine threshold.
1569
+ const longAgo = new Date(Date.now() - 120000);
1570
+ await fs.utimes(filePath, longAgo, longAgo);
1571
+ // A directory already occupying the destination path makes the rename fail.
1572
+ await fs.mkdir(path.join(tmpRoot, 'db', 'unusableReports', '260101T0000-bad.json'), {recursive: true});
1573
+ const savedDbDir = process.env.DB_DIR;
1574
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1575
+ try {
1576
+ await quarantineReport('260101T0000', 'bad', 'not usable');
1577
+ // The report stays where it was.
1578
+ await fs.access(filePath);
1579
+ }
1580
+ finally {
1581
+ if (savedDbDir !== undefined) {
1582
+ process.env.DB_DIR = savedDbDir;
1583
+ }
1584
+ else {
1585
+ delete process.env.DB_DIR;
1586
+ }
1587
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1588
+ }
1589
+ });
1590
+
1591
+ test('getReport quarantines a malformed report file old enough to be quarantined', async () => {
1592
+ const os = await import('node:os');
1593
+ const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'kilotest-getreport-bad-'));
1594
+ const tmpReportsDir = path.join(tmpRoot, 'db', 'reports');
1595
+ await fs.mkdir(tmpReportsDir, {recursive: true});
1596
+ const filePath = path.join(tmpReportsDir, '260101T0000-bad.json');
1597
+ await fs.writeFile(filePath, 'not JSON');
1598
+ // Age the file beyond the quarantine threshold.
1599
+ const longAgo = new Date(Date.now() - 120000);
1600
+ await fs.utimes(filePath, longAgo, longAgo);
1601
+ const savedDbDir = process.env.DB_DIR;
1602
+ process.env.DB_DIR = path.join(tmpRoot, 'db');
1603
+ try {
1604
+ const {getReport} = await import('./util.ts');
1605
+ const result = await getReport('260101T0000', 'bad') as any;
1606
+ assert.ok(result.error);
1607
+ // The malformed file was quarantined.
1608
+ const quarantined = await fs.readdir(path.join(tmpRoot, 'db', 'unusableReports'));
1609
+ assert.deepEqual(quarantined, ['260101T0000-bad.json']);
1610
+ }
1611
+ finally {
1612
+ if (savedDbDir !== undefined) {
1613
+ process.env.DB_DIR = savedDbDir;
1614
+ }
1615
+ else {
1616
+ delete process.env.DB_DIR;
1617
+ }
1618
+ await fs.rm(tmpRoot, {recursive: true, force: true});
1619
+ }
1620
+ });
1621
+
1448
1622
  test('getReportExtracts returns extracts of all available reports', async () => {
1449
1623
  const extracts = await getReportExtracts();
1450
1624
  assert.ok(extracts.length >= 8);
package/util.ts CHANGED
@@ -32,6 +32,9 @@ export const testRequestsPath = (): string => path.join(jobsPath(), 'testRequest
32
32
  export const reportsPath = (): string => path.join(dbPath(), 'reports');
33
33
  // Path of the hidden-reports directory.
34
34
  export const hiddenReportsPath = (): string => path.join(dbPath(), 'hiddenReports');
35
+ // Path of the unusable-reports directory, where reports that fail the usability
36
+ // checks are quarantined for a manager to repair or delete.
37
+ export const unusableReportsPath = (): string => path.join(dbPath(), 'unusableReports');
35
38
  // Path of the usage-metrics file.
36
39
  export const metricsPath = (): string => path.join(dbPath(), 'metrics.json');
37
40
  // IDs, names, and sponsors of Testaro rule engines.
@@ -692,7 +695,49 @@ export const isUsableReport = (report: unknown): report is UsableReport => {
692
695
  && !isNaN(getReportEndTime(r).getTime())
693
696
  && typeof r.catalog === 'object';
694
697
  };
695
- // Returns a report.
698
+ // Minimum age, in milliseconds, of the last modification of a report file for the
699
+ // file to be quarantined. A more recently modified file may still be being written
700
+ // by a worker or a reannotation, so quarantining it could mistake an incomplete write
701
+ // for a defect. Read at call time so tests can shrink it via REPORT_QUARANTINE_MIN_AGE_MS;
702
+ // a value of 0 disables the age check.
703
+ const quarantineMinAgeMs = () => getEnvMax('REPORT_QUARANTINE_MIN_AGE_MS', 60000);
704
+ // Moves a report file from the reports directory to the unusable-reports directory
705
+ // and alerts a manager, if the file exists and is old enough that it can no longer
706
+ // plausibly be mid-write. Called by getReport whenever a report proves defective, so
707
+ // that a defective report is quarantined rather than merely hidden from whichever
708
+ // listing encountered it. Never throws: a failure here must not displace the error
709
+ // that getReport is already reporting.
710
+ export const quarantineReport = async (timeStamp: string, jobID: string, defect: string): Promise<void> => {
711
+ const jobName = `${timeStamp}-${jobID}`;
712
+ // Get the modification time of the report file. A missing file needs no move.
713
+ const stat = await fs.stat(getReportPath(timeStamp, jobID)).catch(() => null);
714
+ if (!stat) {
715
+ return;
716
+ }
717
+ // Leave a recently modified file alone, since it may still be being written.
718
+ if (Date.now() - stat.mtimeMs < quarantineMinAgeMs()) {
719
+ return;
720
+ }
721
+ try {
722
+ await fs.mkdir(unusableReportsPath(), {recursive: true});
723
+ await fs.rename(
724
+ getReportPath(timeStamp, jobID), path.join(unusableReportsPath(), `${jobName}.json`)
725
+ );
726
+ }
727
+ catch (error: unknown) {
728
+ // The report stays excluded regardless, so a failed move is only logged.
729
+ console.error(`ERROR: Failed to quarantine report ${jobName} (${errorMessage(error)})`);
730
+ return;
731
+ }
732
+ // Alert a manager, who can repair the report in place or delete it.
733
+ await sendAlert(
734
+ 'Kilotest: unusable report',
735
+ `Report ${jobName} is ${defect}. It was moved to the unusableReports directory, where a manager can repair or delete it.`
736
+ );
737
+ };
738
+ // Returns a report, or an error object if it is missing, unreadable, unparsable, or
739
+ // unusable. A report that exists but cannot be read or is unusable is defective, so
740
+ // it is quarantined rather than merely rejected by whichever read encountered it.
696
741
  export const getReport = async (timeStamp: string, jobID: string): Promise<UsableReport | {error: string}> => {
697
742
  try {
698
743
  const reportJSON = await fs.readFile(getReportPath(timeStamp, jobID), 'utf8');
@@ -702,11 +747,15 @@ export const getReport = async (timeStamp: string, jobID: string): Promise<Usabl
702
747
  // Return it.
703
748
  return report;
704
749
  }
705
- // Otherwise, i.e. if it is unusable, return this.
706
- return {error: `Report ${timeStamp}-${jobID} is not usable`};
707
750
  } catch (error: unknown) {
751
+ // The file exists but cannot be read or parsed, or is already missing.
752
+ // quarantineReport ignores a missing file, so no existence check is needed here.
753
+ await quarantineReport(timeStamp, jobID, 'unreadable or not JSON');
708
754
  return {error: `Report ${timeStamp}-${jobID} is missing, unreadable, or not JSON (${errorMessage(error)})`};
709
755
  }
756
+ // The report was read but is not usable, so quarantine it.
757
+ await quarantineReport(timeStamp, jobID, 'not usable');
758
+ return {error: `Report ${timeStamp}-${jobID} is not usable`};
710
759
  };
711
760
  // Returns whether a getReport result is an error (rather than a usable report).
712
761
  // A type predicate is used instead of `if (report.error)` because UsableReport
@@ -773,30 +822,27 @@ export const getReportEndTime = (report: {jobData?: {endTime?: string}}): Date =
773
822
  `20${report.jobData?.endTime}Z`
774
823
  );
775
824
 
776
- // Returns an extract of an available report, or an error object if it cannot be read or parsed.
825
+ // Returns an extract of an available report, or an error object if the report cannot
826
+ // be used. An extract exists to expose a report in listings, so it shares getReport's
827
+ // usability bar: a report too defective to expose must produce no extract.
777
828
  export const getReportExtract = async (timeStamp: string, jobID: string): Promise<ReportExtract | {error: string}> => {
778
- try {
779
- // Get the report.
780
- const reportJSON = await fs.readFile(
781
- path.join(reportsPath(), `${timeStamp}-${jobID}.json`), 'utf8'
782
- );
783
- const report = JSON.parse(reportJSON);
784
- const {target, jobData} = report;
785
- const {what: description, url} = target;
786
- // Return an extract of it.
787
- return {
788
- timeStamp,
789
- jobID,
790
- description,
791
- url,
792
- reportTime: getReportEndTime({jobData}).toISOString()
793
- };
794
- }
795
- catch {
796
- return {
797
- error: `No report ${timeStamp}-${jobID} is available`
798
- };
829
+ const report = await getReport(timeStamp, jobID);
830
+ // If the report is not usable:
831
+ if (isReportError(report)) {
832
+ // Return the reason.
833
+ return report;
799
834
  }
835
+ const {target, jobData} = report;
836
+ const {what: description, url} = target;
837
+ // Return an extract of it. Usability has already guaranteed these properties,
838
+ // including a valid completion time.
839
+ return {
840
+ timeStamp,
841
+ jobID,
842
+ description,
843
+ url,
844
+ reportTime: getReportEndTime({jobData}).toISOString()
845
+ };
800
846
  };
801
847
  // Cache of report extracts, keyed by report file path. Each entry records the modification
802
848
  // time and size of the file when it was extracted, so an entry is reused only while the file
@@ -804,7 +850,7 @@ export const getReportExtract = async (timeStamp: string, jobID: string): Promis
804
850
  // written, hidden, or deleted by any process, so the cache is validated against the files
805
851
  // before every use rather than invalidated by the code that changes them.
806
852
  const reportExtractCache = new Map<string, {mtimeMs: number, size: number, extract: ReportExtract}>();
807
- // Returns extracts of all available reports.
853
+ // Returns extracts of all available reports, ordered by page description and completion time.
808
854
  export const getReportExtracts = async (onlyLatest: boolean = false): Promise<ReportExtract[]> => {
809
855
  const reportsDir = reportsPath();
810
856
  // Get the names and paths of the available report files.
@@ -866,8 +912,7 @@ export const getReportExtracts = async (onlyLatest: boolean = false): Promise<Re
866
912
  // Return the array, excluding extracts of superseded reports if so specified.
867
913
  return onlyLatest ? extracts.filter(extract => !extract.superseded) : extracts;
868
914
  };
869
- // Returns extracts of all available reports about the page with a description, from oldest
870
- // to latest.
915
+ // Returns extracts of all available reports about a page, from oldest to latest.
871
916
  export const getPageReportExtracts = async (description: string): Promise<ReportExtract[]> => {
872
917
  const reportExtracts = await getReportExtracts();
873
918
  return reportExtracts.filter(extract => extract.description === description);
@@ -1160,7 +1205,7 @@ export const orderJob = (
1160
1205
  // waits before giving up. Both are read at call time (not cached), via environment
1161
1206
  // variables, so tests can shrink them well below the real ~2-to-4-minute wait a
1162
1207
  // production caller experiences; AWAIT_JOB_TIMEOUT_MS deliberately exceeds the
1163
- // documented "occasionally up to 4 minutes" a caller is told to expect, leaving margin
1208
+ // documented "2 to 4 minutes" a caller is told to expect, leaving margin
1164
1209
  // so a job that is genuinely still running is not mistaken for one that never will
1165
1210
  // finish.
1166
1211
  const awaitJobPollMs = () => getEnvMax('AWAIT_JOB_POLL_MS', 2000);
@@ -16,15 +16,15 @@
16
16
  "sameAs": "https://github.com/jrpool/kilotest",
17
17
  "datePublished": "2025-10-14",
18
18
  "dateModified": "2026-08-26",
19
- "applicationCategory": "Front-end quality (i.e. web accessibility, usability, and standards conformity) testing and reporting service",
19
+ "applicationCategory": "Front-end quality (i.e. accessibility, usability, and web standards) testing and reporting service",
20
20
  "isAccessibleForFree": true,
21
- "description": "Testing of web pages for front-end quality (i.e. web accessibility, usability, and standards conformity) with an ensemble of 12 rule engines and provision of results at selectable levels of detail.",
21
+ "description": "Testing of web pages for front-end quality (i.e. accessibility, usability, and web standards) with an ensemble of 12 rule engines and provision of results at selectable levels of detail.",
22
22
  "softwareHelp": {
23
23
  "@type": "CreativeWork",
24
24
  "url": "https://kilotest.com/tutorialWeb.html"
25
25
  },
26
26
  "featureList": [
27
- "Automated testing of front-end quality (i.e. web accessibility, usability, and standards conformity)",
27
+ "Automated testing of front-end quality (i.e. accessibility, usability, and web standards)",
28
28
  "WCAG 2.2 conformance evaluation",
29
29
  "HTML5 conformance evaluation",
30
30
  "CSS3 conformance evaluation",
@@ -62,7 +62,7 @@
62
62
  <details>
63
63
  <summary>More than a thousand? Really?</summary>
64
64
  <h2>Ensemble testing</h2>
65
- <p>Kilotest does <strong>ensemble testing</strong> of web pages for <a href="https://www.w3.org/mission/accessibility/">accessibility</a>, usability, and standards conformity.</p>
65
+ <p>Kilotest does <strong>ensemble testing</strong> of web pages for <a href="https://www.w3.org/mission/accessibility/">accessibility</a>, usability, and web standards.</p>
66
66
  <p>Efficient testing <strong>first</strong> leverages <strong>automated rule engines</strong> and corrects the defects that they discover, <strong>then</strong> employs human testers.</p>
67
67
  <p>Choosing among the <a href="https://www.w3.org/WAI/test-evaluate/tools/list/">many automated rule engines</a> is hard, because they <a href="https://arxiv.org/pdf/2304.07591">vary in what and how they test</a>.</p>
68
68
  <p><strong>Ensemble testing</strong>, instead of choosing only one, combines <strong>multiple rule engines</strong>. It can discover more issues than single-rule-engine testing, and the rule engines can check one another&rsquo;s quality.</p>
@@ -13,7 +13,7 @@
13
13
  <body>
14
14
  <main>
15
15
  <h1><a href="/">Kilotest</a>: Issues reported for __target__</h1>
16
-
16
+ __oldWarn__
17
17
  <h2>About the tests</h2>
18
18
  __pageFacts__
19
19
  <h2>History</h2>
@@ -123,46 +123,62 @@ const getIssuesData = (report: UsableReport): ResultsSummary & Record<string, an
123
123
  // Return the data.
124
124
  return final;
125
125
  };
126
- // Returns the lines of the history of the reports about the page of a report.
127
- const getHistoryLines = async (
128
- timeStamp: string, jobID: string, description: string, margin: string
129
- ): Promise<string[]> => {
126
+ // Returns data and HTML lines about the history of the reports about the page of a report.
127
+ const getHistory = async (
128
+ timeStamp: string, jobID: string, description: string, margin: string = ''
129
+ ): Promise<{isLatest: boolean, lines: string[]}> => {
130
130
  // Get extracts of all reports about the page, from oldest to latest.
131
131
  const pageExtracts = await getPageReportExtracts(description);
132
- const lastIndex = pageExtracts.length - 1;
133
- // Initialize the list items and whether the report is the latest about the page.
134
- const itemLines: string[] = [];
135
- let isLatest = false;
136
- // For each report about the page:
132
+ // The report's own extract may be absent if the report was quarantined between the
133
+ // report read and the extract listing, so an empty list means "not latest".
134
+ const lastExtract = pageExtracts[pageExtracts.length - 1];
135
+ const isLatest = lastExtract !== undefined
136
+ && lastExtract.timeStamp === timeStamp && lastExtract.jobID === jobID;
137
+ // Initialize the history.
138
+ const history = {
139
+ isLatest,
140
+ lines: [] as string[]
141
+ };
142
+ // For each report about the page, from oldest to latest:
137
143
  pageExtracts.forEach((extract, index) => {
144
+ // Get whether it is this report.
138
145
  const isThis = extract.timeStamp === timeStamp && extract.jobID === jobID;
139
- isLatest ||= isThis && index === lastIndex;
140
146
  // Get the tags that identify its position and whether it is this report.
141
147
  const tags = [
142
148
  index === 0 ? 'oldest' : '',
143
- index === lastIndex ? 'latest' : '',
149
+ index === pageExtracts.length - 1 ? 'latest' : '',
144
150
  isThis ? 'this report' : ''
145
151
  ]
146
152
  .filter(tag => tag);
153
+ // Get a string containing the applicable tags.
147
154
  const tagString = tags.length ? ` (${tags.join('; ')})` : '';
148
155
  const label = `${getAgoString(new Date(extract.reportTime))} ago${tagString}`;
149
- // Add a list item describing it, linked unless it is this report, to the items.
150
- const href = `/listIssues.html/${extract.timeStamp}/${extract.jobID}`;
151
- const content = isThis ? label : `<a href="${href}">${label}</a>`;
152
- itemLines.push(`${margin} <li>${content}</li>`);
156
+ let content;
157
+ if (isThis) {
158
+ content = label;
159
+ }
160
+ else {
161
+ const href = `/listIssues.html/${extract.timeStamp}/${extract.jobID}`;
162
+ content = `<a href="${href}">${label}</a>`;
163
+ }
164
+ // Add a list item describing the report, linked if not this report, to the history.
165
+ history.lines.push(`${margin}<li>${content}</li>`);
153
166
  });
154
167
  // Initialize the lines with a list of the reports, or a statement that there is only one.
155
- const lines = itemLines.length > 1
156
- ? [`${margin}<ol>`, ...itemLines, `${margin}</ol>`]
157
- : [`${margin}<p>This is the only report about the page.</p>`];
158
- // If the report is the latest about the page:
168
+ const lines = history.lines.length > 1
169
+ ? [`${margin}<ol>`, ...history.lines, `${margin}</ol>`]
170
+ : [`${margin}<p>This is the only available report about the page.</p>`];
171
+ // If this report is the latest about the page:
159
172
  if (isLatest) {
160
173
  // Add a link to the form for requesting a retest to the lines.
161
- const href = `/requestRetestForm.html/${timeStamp}/${jobID}`;
162
- lines.push(`${margin}<p><a href="${href}">Should Kilotest retest the page?</a></p>`);
174
+ lines.push(
175
+ `${margin}<p><a href="/requestRetestForm.html/${timeStamp}/${jobID}">Request a retest</a></p>`
176
+ );
163
177
  }
164
- // Return the lines.
165
- return lines;
178
+ // Replace the bare list items with the assembled section lines.
179
+ history.lines = lines;
180
+ // Return the history.
181
+ return history;
166
182
  };
167
183
  // Adds parameters to a query for the answer page.
168
184
  const populateQuery = async (timeStamp: string, jobID: string, query: Record<string, any>) => {
@@ -191,10 +207,15 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record<str
191
207
  query.target = description;
192
208
  query.testInfo = testInfo;
193
209
  const margin = ' '.repeat(6);
210
+ const history = await getHistory(timeStamp, jobID, description, ' '.repeat(8));
211
+ // Add a warning if the report is not the latest about the page.
212
+ query.oldWarn = history.isLatest
213
+ ? ''
214
+ : '<p><strong>Warning: This report is not the latest about the page.</strong> See history below.</p>';
194
215
  // Add the page-facts list to the query.
195
216
  query.pageFacts = getPageFactsLines(pageInfo, issuesData, margin).join('\n');
196
217
  // Add the history of reports about the page to the query.
197
- query.history = (await getHistoryLines(timeStamp, jobID, description, margin)).join('\n');
218
+ query.history = history.lines.join('\n');
198
219
  const {issues} = issuesData;
199
220
  // Add a summary of the issues to the query.
200
221
  query.highestCount = issues[4].length;
@@ -231,7 +231,7 @@ test('listIssues History of an older report lists all reports, with this one unl
231
231
  assert.equal(items[0]!.href, null);
232
232
  assert.match(items[1]!.text, /^\d+ days? ago \(latest\)$/);
233
233
  assert.equal(items[1]!.href, '/listIssues.html/260202T0000/new');
234
- assert.ok(!result.answerPage.includes('Should Kilotest retest the page?'));
234
+ assert.ok(!result.answerPage.includes('Request a retest'));
235
235
  });
236
236
 
237
237
  test('listIssues History of the latest report ends with the retest question', async () => {
@@ -244,7 +244,7 @@ test('listIssues History of the latest report ends with the retest question', as
244
244
  assert.match(items[1]!.text, /^\d+ days? ago \(latest; this report\)$/);
245
245
  assert.equal(items[1]!.href, null);
246
246
  const retestLink = history.querySelector('ol + p > a');
247
- assert.equal(retestLink?.text, 'Should Kilotest retest the page?');
247
+ assert.equal(retestLink?.text, 'Request a retest');
248
248
  assert.equal(retestLink?.getAttribute('href'), '/requestRetestForm.html/260202T0000/new');
249
249
  });
250
250
 
@@ -253,7 +253,7 @@ test('listIssues History of the only report about a page says so and asks about
253
253
  const history = getHistory(result.answerPage);
254
254
  assert.equal(getHistoryItems(history).length, 0);
255
255
  const paragraphs = history.querySelectorAll('p');
256
- assert.equal(paragraphs[0]?.text, 'This is the only report about the page.');
256
+ assert.equal(paragraphs[0]?.text, 'This is the only available report about the page.');
257
257
  assert.equal(paragraphs[1]?.querySelector('a')?.getAttribute('href'), '/requestRetestForm.html/260101T0001/ct');
258
258
  });
259
259
 
@@ -149,12 +149,12 @@ test('listPages shows queued and claimed jobs when they exist', async () => {
149
149
  }
150
150
  });
151
151
 
152
- test('listPages lists a page whose report is not usable', {timeout: 500}, async () => {
152
+ test('listPages omits a page whose report is not usable', {timeout: 500}, async () => {
153
153
  const dbDir = (await import('../../test/dbFixture.ts')).fixtureDBDir;
154
154
  const reportsDir = path.join(dbDir, 'reports');
155
155
  const invalidReportPath = path.join(reportsDir, '260101T9999-bad.json');
156
156
  try {
157
- // Valid for getReportExtract (has target and jobData.endTime) but not usable by isUsableReport (no acts or catalog).
157
+ // Not usable by isUsableReport (no acts or catalog), so it must not be exposed.
158
158
  const invalidReport = {
159
159
  target: {what: 'Invalid Report', url: 'https://example.com/invalid'},
160
160
  jobData: {endTime: '26-01-01T00:10'}
@@ -163,7 +163,7 @@ test('listPages lists a page whose report is not usable', {timeout: 500}, async
163
163
  const result: any = await answer();
164
164
  assert.equal(result.status, 'ok');
165
165
  const links = getPageLinks(result.answerPage);
166
- assert.ok(links.some(a => a.text === 'Invalid Report'));
166
+ assert.ok(!links.some(a => a.text === 'Invalid Report'));
167
167
  }
168
168
  finally {
169
169
  await fs.unlink(invalidReportPath).catch(() => {});