@jrpool/kilotest 46.0.2 → 46.0.3

package/openapi.yaml

@@ -254,9 +254,9 @@ paths:
                                       type: object
                                       properties:
                                         name:
-                                          anyOf:
-                                            - type: string
-                                            - type: 'null'
+                                          type:
+                                            - string
+                                            - 'null'
                                         reason for failure:
                                           description: Copied verbatim from the Testaro report; shape may vary.
                                       required:
@@ -266,9 +266,9 @@ paths:
                                   names of rule engines that reported rule violations:
                                     type: array
                                     items:
-                                      anyOf:
-                                        - type: string
-                                        - type: 'null'
+                                      type:
+                                        - string
+                                        - 'null'
                                   counts of issues by priority:
                                     type: object
                                     properties:
@@ -782,9 +782,9 @@ paths:
                     type: object
                     properties:
                       size of the report in bytes:
-                        anyOf:
-                          - type: number
-                          - type: string
+                        type:
+                          - number
+                          - string
                       full report:
                         description: The full raw Testaro/Testilo report JSON, copied verbatim, or an error object if it could not be retrieved.
                     required:
@@ -1224,9 +1224,9 @@ components:
           type: string
           description: URL of the equivalent web UI page for this request.
         closest ancestor request:
-          anyOf:
-            - type: string
-            - type: 'null'
+          type:
+            - string
+            - 'null'
       required:
         - this request
         - closest ancestor request
@@ -1290,13 +1290,13 @@ components:
         identifier:
           type: string
         name:
-          anyOf:
-            - type: string
-            - type: 'null'
+          type:
+            - string
+            - 'null'
         sponsor:
-          anyOf:
-            - type: string
-            - type: 'null'
+          type:
+            - string
+            - 'null'
       required:
         - identifier
         - name

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "46.0.2",
+  "version": "46.0.3",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "engines": {

package/test/fixtures/buildFixtures.js

@@ -0,0 +1,244 @@
+/*
+  buildFixtures.js
+  Builds the fixture corpus for Kilotest tests.
+
+  Each fixture is a minimal Testaro report in the current format, crafted so
+  that the expected API response can be hand-computed and hard-coded into
+  tests. The fixtures collectively cover the `outcome` property values
+  (`failed`, `cantTell`, and missing/undefined), superseded reports, empty
+  reports, and prevented rule engines.
+
+  Run with: node test/fixtures/buildFixtures.js [targetDir]
+  Defaults to test/fixtures/db under the project root.
+*/
+
+// IMPORTS
+
+const fs = require('fs/promises');
+const path = require('path');
+
+// CONSTANTS
+
+// Issue IDs with known specs in testaro-issues, chosen for different weights.
+const ISSUE_LINK_NO_TEXT = 'linkNoText';       // weight 4 (highest)
+const ISSUE_FOCUS_INDICATION = 'focusIndicationBad'; // weight 4
+const ISSUE_ALL_CAPS = 'allCaps';              // weight 1 (lowest)
+
+// Rule engine IDs that exist in util.js ruleEngines.
+const ENGINE_AXE = 'axe';
+const ENGINE_ALFA = 'alfa';
+const ENGINE_IBM = 'ibm';
+
+// FUNCTIONS
+
+// Returns a minimal catalog item.
+const catalogItem = (tagName, text, pathID = '/html', boxID = '0:0:100:50') => ({
+  tagName,
+  id: '',
+  startTag: `<${tagName.toLowerCase()}>`,
+  text,
+  textLinkable: false,
+  boxID,
+  pathID,
+  headingIndex: '',
+  checkpoint: 0
+});
+
+// Returns a minimal standard instance.
+const instance = (
+  ruleID, what, outcome, issueID, catalogIndex, ordinalSeverity = 2, count = 1
+) => ({
+  ruleID,
+  what,
+  ordinalSeverity,
+  outcome,
+  count,
+  catalogIndex: String(catalogIndex),
+  checkpoint: 0,
+  issueID
+});
+
+// Returns a minimal test act.
+const testAct = (which, instances) => ({
+  type: 'test',
+  which,
+  startTime: '26-01-01T00:00',
+  endTime: '26-01-01T00:01',
+  result: {
+    standardResult: {
+      instances,
+      outcomeTotals: {
+        failed: instances.filter(i => i.outcome === 'failed').length,
+        cantTell: instances.filter(i => i.outcome === 'cantTell').length
+      }
+    }
+  }
+});
+
+// Returns a minimal valid report.
+const report = (id, what, url, acts, catalog, endTime = '26-01-01T00:10') => ({
+  id,
+  what,
+  strict: false,
+  standard: 'only',
+  device: {id: 'default'},
+  browserID: 'chromium',
+  creationTimeStamp: id.slice(0, 11),
+  executionTimeStamp: id.slice(0, 11),
+  target: {what, url},
+  sources: {worker: 'test-worker'},
+  acts,
+  jobData: {
+    startTime: '26-01-01T00:00',
+    endTime,
+    elapsedSeconds: 600,
+    preventions: {},
+    issuelessRules: []
+  },
+  catalog,
+  images: {},
+  checkpoints: []
+});
+
+// Writes a JSON file with a trailing newline.
+const writeJSON = async (filePath, object) => {
+  await fs.writeFile(filePath, `${JSON.stringify(object, null, 2)}\n`);
+};
+
+// MAIN
+
+const main = async () => {
+  const targetDir = process.argv[2] || path.join(__dirname, 'db');
+  const reportsDir = path.join(targetDir, 'reports');
+  await fs.mkdir(reportsDir, {recursive: true});
+  await fs.mkdir(path.join(targetDir, 'jobs', 'queue'), {recursive: true});
+  await fs.mkdir(path.join(targetDir, 'jobs', 'claimed'), {recursive: true});
+  await fs.mkdir(path.join(targetDir, 'jobs', 'failed'), {recursive: true});
+  await fs.mkdir(path.join(targetDir, 'hiddenReports'), {recursive: true});
+
+  // Fixture 1: mixedOutcomes
+  // Two rule engines, three instances: one failed, one cantTell, one failed.
+  // The cantTell instance must be excluded from issue/violator counts.
+  // Two distinct issues (linkNoText weight 4, allCaps weight 1) and two
+  // distinct violator catalog indexes (0 and 1).
+  const mixedCatalog = {
+    '0': catalogItem('A', 'About Us', '/html/body/a[1]', '10:20:80:30'),
+    '1': catalogItem('P', 'ALL ABOUT US', '/html/body/p[1]', '10:60:80:20')
+  };
+  const mixedActs = [
+    testAct(ENGINE_AXE, [
+      instance('r11', 'The link does not have an accessible name', 'failed',
+        ISSUE_LINK_NO_TEXT, 0, 2, 1),
+      instance('r65', 'Focus Visible', 'cantTell',
+        ISSUE_FOCUS_INDICATION, 0, 0, 1)
+    ]),
+    testAct(ENGINE_ALFA, [
+      instance('r11', 'The link does not have an accessible name', 'failed',
+        ISSUE_LINK_NO_TEXT, 0, 2, 1),
+      instance('r3', 'Text is all-capital', 'failed',
+        ISSUE_ALL_CAPS, 1, 1, 1)
+    ])
+  ];
+  await writeJSON(
+    path.join(reportsDir, '260101T0000-mix.json'),
+    report('260101T0000-mix', 'Mixed Outcomes Page',
+      'https://example.com/mixed', mixedActs, mixedCatalog)
+  );
+
+  // Fixture 2: allCantTell
+  // All instances have outcome cantTell, so no issues or violators should be
+  // reported by listIssues, listViolators, or listDiagnoses.
+  const cantTellCatalog = {
+    '0': catalogItem('A', 'Click here', '/html/body/a[1]', '5:10:60:20')
+  };
+  const cantTellActs = [
+    testAct(ENGINE_AXE, [
+      instance('r65', 'Focus Visible', 'cantTell',
+        ISSUE_FOCUS_INDICATION, 0, 0, 1)
+    ])
+  ];
+  await writeJSON(
+    path.join(reportsDir, '260101T0001-ct.json'),
+    report('260101T0001-ct', 'All CantTell Page',
+      'https://example.com/canttell', cantTellActs, cantTellCatalog)
+  );
+
+  // Fixture 3: noOutcomes
+  // Instances with no outcome property. Testaro defaults to 'failed', but
+  // Kilotest code checks `outcome !== 'cantTell'`, so missing outcome (undefined)
+  // is treated as a violation. This verifies that behavior.
+  const noOutcomeCatalog = {
+    '0': catalogItem('BUTTON', 'Submit', '/html/body/button[1]', '15:25:70:30')
+  };
+  const noOutcomeActs = [
+    testAct(ENGINE_IBM, [
+      {
+        ruleID: 'r1',
+        what: 'Button has no accessible name',
+        ordinalSeverity: 3,
+        count: 1,
+        catalogIndex: '0',
+        checkpoint: 0,
+        issueID: ISSUE_LINK_NO_TEXT
+      }
+    ])
+  ];
+  await writeJSON(
+    path.join(reportsDir, '260101T0002-no.json'),
+    report('260101T0002-no', 'No Outcomes Page',
+      'https://example.com/nooutcomes', noOutcomeActs, noOutcomeCatalog)
+  );
+
+  // Fixture 4: superseded
+  // A report about the same page as mixedOutcomes but with a later timestamp,
+  // so mixedOutcomes is superseded by this one.
+  const newerActs = [
+    testAct(ENGINE_AXE, [
+      instance('r11', 'The link does not have an accessible name', 'failed',
+        ISSUE_LINK_NO_TEXT, 0, 2, 1)
+    ])
+  ];
+  await writeJSON(
+    path.join(reportsDir, '260202T0000-new.json'),
+    report('260202T0000-new', 'Mixed Outcomes Page',
+      'https://example.com/mixed', newerActs, mixedCatalog, '26-02-02T00:10')
+  );
+
+  // Fixture 5: empty
+  // A valid report with no test acts that have instances, so listIssues
+  // returns zero issues.
+  await writeJSON(
+    path.join(reportsDir, '260101T0005-emp.json'),
+    report('260101T0005-emp', 'Empty Results Page',
+      'https://example.com/empty', [testAct(ENGINE_AXE, [])], {})
+  );
+
+  // Fixture 6: prevented
+  // A report where one rule engine was prevented from testing.
+  const preventedCatalog = {
+    '0': catalogItem('IMG', 'An image', '/html/body/img[1]', '0:0:200:100')
+  };
+  const preventedActs = [
+    testAct(ENGINE_AXE, [
+      instance('r11', 'The image has no alt text', 'failed',
+        ISSUE_LINK_NO_TEXT, 0, 2, 1)
+    ])
+  ];
+  const preventedReport = report('260101T0006-prv', 'Prevented Page',
+    'https://example.com/prevented', preventedActs, preventedCatalog);
+  preventedReport.jobData.preventions = {alfa: 'page timed out'};
+  await writeJSON(
+    path.join(reportsDir, '260101T0006-prv.json'),
+    preventedReport
+  );
+
+  // Write an empty recs.json so getRecs does not try to create one.
+  await writeJSON(path.join(targetDir, 'jobs', 'recs.json'), {});
+
+  console.log(`Fixtures built in ${targetDir}`);
+};
+
+main().catch(error => {
+  console.error(error);
+  process.exit(1);
+});

package/test/fixtures/db/jobs/recs.json

@@ -0,0 +1 @@
+{}

package/test/fixtures/db/reports/260101T0000-mix.json

@@ -0,0 +1,126 @@
+{
+  "id": "260101T0000-mix",
+  "what": "Mixed Outcomes Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260101T0000",
+  "executionTimeStamp": "260101T0000",
+  "target": {
+    "what": "Mixed Outcomes Page",
+    "url": "https://example.com/mixed"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "axe",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r11",
+              "what": "The link does not have an accessible name",
+              "ordinalSeverity": 2,
+              "outcome": "failed",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "linkNoText"
+            },
+            {
+              "ruleID": "r65",
+              "what": "Focus Visible",
+              "ordinalSeverity": 0,
+              "outcome": "cantTell",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "focusIndicationBad"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 1,
+            "cantTell": 1
+          }
+        }
+      }
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r11",
+              "what": "The link does not have an accessible name",
+              "ordinalSeverity": 2,
+              "outcome": "failed",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "linkNoText"
+            },
+            {
+              "ruleID": "r3",
+              "what": "Text is all-capital",
+              "ordinalSeverity": 1,
+              "outcome": "failed",
+              "count": 1,
+              "catalogIndex": "1",
+              "checkpoint": 0,
+              "issueID": "allCaps"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 2,
+            "cantTell": 0
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-01-01T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {},
+    "issuelessRules": []
+  },
+  "catalog": {
+    "0": {
+      "tagName": "A",
+      "id": "",
+      "startTag": "<a>",
+      "text": "About Us",
+      "textLinkable": false,
+      "boxID": "10:20:80:30",
+      "pathID": "/html/body/a[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    },
+    "1": {
+      "tagName": "P",
+      "id": "",
+      "startTag": "<p>",
+      "text": "ALL ABOUT US",
+      "textLinkable": false,
+      "boxID": "10:60:80:20",
+      "pathID": "/html/body/p[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    }
+  },
+  "images": {},
+  "checkpoints": []
+}

package/test/fixtures/db/reports/260101T0001-ct.json

@@ -0,0 +1,69 @@
+{
+  "id": "260101T0001-ct",
+  "what": "All CantTell Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260101T0001",
+  "executionTimeStamp": "260101T0001",
+  "target": {
+    "what": "All CantTell Page",
+    "url": "https://example.com/canttell"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "axe",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r65",
+              "what": "Focus Visible",
+              "ordinalSeverity": 0,
+              "outcome": "cantTell",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "focusIndicationBad"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 0,
+            "cantTell": 1
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-01-01T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {},
+    "issuelessRules": []
+  },
+  "catalog": {
+    "0": {
+      "tagName": "A",
+      "id": "",
+      "startTag": "<a>",
+      "text": "Click here",
+      "textLinkable": false,
+      "boxID": "5:10:60:20",
+      "pathID": "/html/body/a[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    }
+  },
+  "images": {},
+  "checkpoints": []
+}

package/test/fixtures/db/reports/260101T0002-no.json

@@ -0,0 +1,68 @@
+{
+  "id": "260101T0002-no",
+  "what": "No Outcomes Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260101T0002",
+  "executionTimeStamp": "260101T0002",
+  "target": {
+    "what": "No Outcomes Page",
+    "url": "https://example.com/nooutcomes"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "ibm",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r1",
+              "what": "Button has no accessible name",
+              "ordinalSeverity": 3,
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "linkNoText"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 0,
+            "cantTell": 0
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-01-01T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {},
+    "issuelessRules": []
+  },
+  "catalog": {
+    "0": {
+      "tagName": "BUTTON",
+      "id": "",
+      "startTag": "<button>",
+      "text": "Submit",
+      "textLinkable": false,
+      "boxID": "15:25:70:30",
+      "pathID": "/html/body/button[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    }
+  },
+  "images": {},
+  "checkpoints": []
+}

package/test/fixtures/db/reports/260101T0005-emp.json

@@ -0,0 +1,46 @@
+{
+  "id": "260101T0005-emp",
+  "what": "Empty Results Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260101T0005",
+  "executionTimeStamp": "260101T0005",
+  "target": {
+    "what": "Empty Results Page",
+    "url": "https://example.com/empty"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "axe",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [],
+          "outcomeTotals": {
+            "failed": 0,
+            "cantTell": 0
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-01-01T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {},
+    "issuelessRules": []
+  },
+  "catalog": {},
+  "images": {},
+  "checkpoints": []
+}

package/test/fixtures/db/reports/260101T0006-prv.json

@@ -0,0 +1,71 @@
+{
+  "id": "260101T0006-prv",
+  "what": "Prevented Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260101T0006",
+  "executionTimeStamp": "260101T0006",
+  "target": {
+    "what": "Prevented Page",
+    "url": "https://example.com/prevented"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "axe",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r11",
+              "what": "The image has no alt text",
+              "ordinalSeverity": 2,
+              "outcome": "failed",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "linkNoText"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 1,
+            "cantTell": 0
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-01-01T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {
+      "alfa": "page timed out"
+    },
+    "issuelessRules": []
+  },
+  "catalog": {
+    "0": {
+      "tagName": "IMG",
+      "id": "",
+      "startTag": "<img>",
+      "text": "An image",
+      "textLinkable": false,
+      "boxID": "0:0:200:100",
+      "pathID": "/html/body/img[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    }
+  },
+  "images": {},
+  "checkpoints": []
+}

package/test/fixtures/db/reports/260202T0000-new.json

@@ -0,0 +1,80 @@
+{
+  "id": "260202T0000-new",
+  "what": "Mixed Outcomes Page",
+  "strict": false,
+  "standard": "only",
+  "device": {
+    "id": "default"
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "260202T0000",
+  "executionTimeStamp": "260202T0000",
+  "target": {
+    "what": "Mixed Outcomes Page",
+    "url": "https://example.com/mixed"
+  },
+  "sources": {
+    "worker": "test-worker"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "which": "axe",
+      "startTime": "26-01-01T00:00",
+      "endTime": "26-01-01T00:01",
+      "result": {
+        "standardResult": {
+          "instances": [
+            {
+              "ruleID": "r11",
+              "what": "The link does not have an accessible name",
+              "ordinalSeverity": 2,
+              "outcome": "failed",
+              "count": 1,
+              "catalogIndex": "0",
+              "checkpoint": 0,
+              "issueID": "linkNoText"
+            }
+          ],
+          "outcomeTotals": {
+            "failed": 1,
+            "cantTell": 0
+          }
+        }
+      }
+    }
+  ],
+  "jobData": {
+    "startTime": "26-01-01T00:00",
+    "endTime": "26-02-02T00:10",
+    "elapsedSeconds": 600,
+    "preventions": {},
+    "issuelessRules": []
+  },
+  "catalog": {
+    "0": {
+      "tagName": "A",
+      "id": "",
+      "startTag": "<a>",
+      "text": "About Us",
+      "textLinkable": false,
+      "boxID": "10:20:80:30",
+      "pathID": "/html/body/a[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    },
+    "1": {
+      "tagName": "P",
+      "id": "",
+      "startTag": "<p>",
+      "text": "ALL ABOUT US",
+      "textLinkable": false,
+      "boxID": "10:60:80:20",
+      "pathID": "/html/body/p[1]",
+      "headingIndex": "",
+      "checkpoint": 0
+    }
+  },
+  "images": {},
+  "checkpoints": []
+}