@jrpool/kilotest 45.0.4 → 46.0.0

package/api/listDiagnoses.js

@@ -6,7 +6,7 @@
 // IMPORTS
 
 const {
-  getIssueClassification,
+  getIssueSpec,
   getReportBasics,
   getResponseMetadata,
   getToolsFacts,
@@ -41,10 +41,10 @@ exports.response = async args => {
     // Add them to the response content.
     responseContent['basics about the report'] = reportBasics;
   }
-  // Get the classification of the issue.
-  const issueClassification = issueID ? getIssueClassification(issueID) : null;
+  // Get the specification of the issue.
+  const issueSpec = issueID ? getIssueSpec(issueID) : null;
   // If the issue is ignorable or not fully classified:
-  if (!issueClassification) {
+  if (!issueSpec) {
     // Add this to the response content.
     responseContent['basics about the issue'] = {
       'error': 'No information about the specified issue is available'
@@ -55,7 +55,7 @@ exports.response = async args => {
   }
   // Otherwise, i.e. if it is non-ignorable and fully classified:
   else {
-    const {summary, wcag, weight, why} = issueClassification;
+    const {summary, wcag, weight, why} = issueSpec;
     // Initialize the basics about it.
     const issueBasics = {
       'identifier': issueID,
@@ -99,7 +99,7 @@ exports.response = async args => {
       // Add them to the response content.
       responseContent['details about the element'] = violatorDetails;
       // If the issue is non-ignorable and fully classified:
-      if (issueClassification) {
+      if (issueSpec) {
         // Initialize data about the diagnoses of the violation of the issue.
         const diagnoses = [];
         // For each act in the report:
@@ -110,9 +110,13 @@ exports.response = async args => {
           if (type === 'test' && which) {
             // For each standard instance of the act:
             instances.forEach(instance => {
-              const {count, ordinalSeverity, ruleID, what} = instance;
-              // If the instance has the issue ID and the catalog index of the violator:
-              if (instance.issueID === issueID && instance.catalogIndex === catalogIndex) {
+              const {count, ordinalSeverity, outcome, ruleID, what} = instance;
+              // If the instance reports a violation of the issue by the violator:
+              if (
+                outcome !== 'cantTell'
+                &&  instance.issueID === issueID
+                &&  instance.catalogIndex === catalogIndex
+              ) {
                 // Get the diagnosis.
                 const diagnosis = {
                   'identifier of the violated rule': ruleID !== what ? ruleID : null,

package/api/listIssues.js

@@ -6,7 +6,7 @@
 // IMPORTS
 
 const {
-  getIssueClassification,
+  getIssueSpec,
   getReportBasics,
   getResponseMetadata,
   getRuleEngineFacts,
@@ -75,29 +75,32 @@ exports.response = async args => {
           const instances = result?.standardResult?.instances ?? [];
           // For each standard instance of the act:
           instances.forEach(instance => {
-            const {catalogIndex, issueID} = instance;
-            // Get the classification of its issue.
-            const issueClassification = issueID ? getIssueClassification(issueID) : null;
-            // If the instance has a non-ignorable and fully classified issue:
-            if (issueClassification) {
-              const {summary, weight, why} = issueClassification;
-              // Ensure the rule-engine ID is in the reporters data.
-              reporterIDs.add(which);
-              // If the instance has a catalog index:
-              if (catalogIndex) {
-                // Ensure the index of the violator is in the results data.
-                violatorIndexes.add(catalogIndex);
+            const {catalogIndex, issueID, outcome} = instance;
+            // If the instance reports a violation:
+            if (outcome !== 'cantTell') {
+              // Get the specification of its issue.
+              const issueSpec = issueID ? getIssueSpec(issueID) : null;
+              // If the instance has a non-ignorable and fully classified issue:
+              if (issueSpec) {
+                const {summary, weight, why} = issueSpec;
+                // Ensure the rule-engine ID is in the reporters data.
+                reporterIDs.add(which);
+                // If the instance has a catalog index:
+                if (catalogIndex) {
+                  // Ensure the index of the violator is in the results data.
+                  violatorIndexes.add(catalogIndex);
+                }
+                // Ensure the data about the issue are in the results data.
+                issuesData[issueID] ??= {
+                  id: issueID,
+                  summary,
+                  weight,
+                  why,
+                  reporterIDs: new Set()
+                };
+                // Ensure the reporter ID is in the data about the issue.
+                issuesData[issueID].reporterIDs.add(which);
               }
-              // Ensure the data about the issue are in the results data.
-              issuesData[issueID] ??= {
-                id: issueID,
-                summary,
-                weight,
-                why,
-                reporterIDs: new Set()
-              };
-              // Ensure the reporter ID is in the data about the issue.
-              issuesData[issueID].reporterIDs.add(which);
             }
           });
         }

package/api/listViolators.js

@@ -6,7 +6,7 @@
 // IMPORTS
 
 const {
-  getIssueClassification,
+  getIssueSpec,
   getReportBasics,
   getResponseMetadata,
   getRuleEnginesFacts,
@@ -41,10 +41,10 @@ exports.response = async args => {
     // Add them to the response content.
     responseContent['basics about the report'] = reportBasics;
   }
-  // Get the classification of the issue.
-  const issueClassification = issueID ? getIssueClassification(issueID) : null;
+  // Get the specification of the issue.
+  const issueSpec = issueID ? getIssueSpec(issueID) : null;
   // If the issue is ignorable or not fully classified:
-  if (!issueClassification) {
+  if (!issueSpec) {
     // Add this to the response content.
     responseContent['basics about the issue'] = {
       'error': 'No information about the specified issue is available'
@@ -52,7 +52,7 @@ exports.response = async args => {
   }
   // Otherwise, i.e. if it is non-ignorable and fully classified:
   else {
-    const {summary, wcag, weight, why} = issueClassification;
+    const {summary, wcag, weight, why} = issueSpec;
     // Initialize the basics and details about it.
     const issueBasics = {
       'identifier': issueID,
@@ -81,18 +81,21 @@ exports.response = async args => {
           instances.forEach(instance => {
             // If the instance has the issue ID:
             if (instance.issueID === issueID) {
-              const {catalogIndex} = instance;
-              // Ensure the rule-engine ID is in the reporters data.
-              reporterIDs.add(which);
-              // If the instance has a catalog index:
-              if (catalogIndex) {
-                // Ensure the catalog index is in the violators data.
-                violators[catalogIndex] ??= {
-                  catalogIndex,
-                  reporters: new Set()
-                };
-                // Ensure the reporter ID is in the violator data.
-                violators[catalogIndex].reporters.add(which);
+              const {catalogIndex, outcome} = instance;
+              // If the instance reports a violation:
+              if (outcome !== 'cantTell') {
+                // Ensure the rule-engine ID is in the reporters data.
+                reporterIDs.add(which);
+                // If the instance has a catalog index:
+                if (catalogIndex) {
+                  // Ensure the catalog index is in the violators data.
+                  violators[catalogIndex] ??= {
+                    catalogIndex,
+                    reporters: new Set()
+                  };
+                  // Ensure the reporter ID is in the data about the violator.
+                  violators[catalogIndex].reporters.add(which);
+                }
               }
             }
           });

package/api/requestRetest.js

@@ -6,7 +6,7 @@
 // IMPORTS
 
 const {getResponseMetadata, getToolsFacts, processTestRequest, thisHost} = require('./util');
-const {getReportsExtract} = require('../util');
+const {getLatestReportExtracts, getReportsExtract} = require('../util');
 
 // FUNCTIONS
 
@@ -35,10 +35,8 @@ exports.response = async args => {
   }
   // Otherwise, if the report has been superseded:
   else if (
-    Object
-    .values(reportsExtract)
-    .filter(extract => extract.what === what && extract.url === url)
-    .length > 1
+    (await getLatestReportExtracts())
+    .every(extract => extract.timeStamp !== timeStamp || extract.jobID !== jobID)
   ) {
     // Add this to the response content.
     responseContent['details about your request'] = {

package/api/testSuite.js

@@ -137,6 +137,8 @@ const requestService = async () => {
   const reportsBasicsIndex = Math.floor(reportsBasics.length * Math.random());
   // Choose one report at random.
   reportBasics = reportsBasics[reportsBasicsIndex];
+  // Get whether the report has been superseded by a later report about the same page.
+  const isSuperseded = reportBasics['whether a later report about the same page exists'];
   [timeStamp, jobID] = reportBasics.identifier?.split('-') || [null, null];
   if (!(timeStamp && jobID)) {
     console.log(`reportBasicsIndex: ${reportsBasicsIndex}`);
@@ -263,13 +265,24 @@ const requestService = async () => {
   responseContent = body?.['response content'] ?? {};
   requestDetails = responseContent['details about your request'] ?? {};
   requestDisposition = responseContent['disposition of your request'] ?? {};
-  if (
-    !requestDetails['reason why the page should be retested']
-    || !requestDisposition['how you can check for completion']
-  ) {
-    console.log(`requestDetails: ${JSON.stringify(requestDetails, null, 2)}`);
-    console.log(`requestDisposition: ${JSON.stringify(requestDisposition, null, 2)}`);
-    return;
+  // If the report has been superseded:
+  if (isSuperseded) {
+    // Expect the request to be rejected.
+    if (!requestDetails.error?.includes('a later report about the page exists')) {
+      console.log(`requestDetails: ${JSON.stringify(requestDetails, null, 2)}`);
+      return;
+    }
+  }
+  // Otherwise, i.e. if the report has not been superseded:
+  else {
+    if (
+      !requestDetails['reason why the page should be retested']
+      || !requestDisposition['how you can check for completion']
+    ) {
+      console.log(`requestDetails: ${JSON.stringify(requestDetails, null, 2)}`);
+      console.log(`requestDisposition: ${JSON.stringify(requestDisposition, null, 2)}`);
+      return;
+    }
   }
   console.log('======================\nRequest: Make a feature request');
   path = '/api/requestFeature';

package/api/util.js

@@ -14,11 +14,11 @@ const {
   getRandomString,
   getReportExtract,
   getReportStats,
-  issuesClassification,
   objectSort,
   ruleEngines,
   updateRecs
 } = require('../util');
+const {issues: issueSpecs} = require('testaro-issues');
 
 // CONSTANTS
 
@@ -106,25 +106,22 @@ exports.getReportBasics = async (timeStamp, jobID) => {
   // Return them.
   return basics;
 };
-// Returns the classification of an issue.
-exports.getIssueClassification = issueID => {
-  // Get the issue classification.
-  const issueClassification = issuesClassification[issueID] ?? {};
-  const {summary, wcag, weight, why} = issueClassification;
-  // If the issue is non-ignorable and fully classified:
-  if (
-    issueID
-    && issueID !== 'ignorable'
-    && issueClassification
-    && summary
-    && wcag
-    && [1, 2, 3, 4].includes(weight)
-    && why
-  ) {
-    // Return the classification.
-    return issueClassification;
+// Returns the specification of an issue.
+exports.getIssueSpec = issueID => {
+  // Get the issue specification.
+  const issueSpec = issueSpecs[issueID];
+  // If it exists:
+  if (issueSpec) {
+    const {summary, wcag, weight, why} = issueSpec;
+    // If the issue is non-ignorable and fully classified:
+    if (issueID !== 'ignorable' && summary && wcag && [1, 2, 3, 4].includes(weight) && why) {
+      // Return its specification.
+      return issueSpec;
+    }
+    // Otherwise, return this.
+    return null;
   }
-  // Otherwise, i.e. if it is ignorable or not fully classified, return this.
+  // Otherwise, i.e. if it does not exist, return this.
   return null;
 };
 // Processes a test or retest request.

package/docs/IDEAS.md

@@ -5858,3 +5858,70 @@ The gamble is **not** on whether the concept is sound (it is), but on **executio
 
 #### Summary
 Your architectural shift from static digests to interactive FAQ + LLM-driven Q&A is **well-grounded in current practice** and **solves real UX problems**. The risks are manageable with proper scoping and validation. I'd recommend a **phased rollout** starting with FAQs + manual custom-question handling, then adding LLM assistance once you understand user intent patterns.
+
+## Testing
+
+Now let's check the testSuite.js more fully to see how it picks fixtures (confirming the deficiency the user described), and check schemas.js/util.js in api for outcome handling.
+
+Here is a plan for building a rigorous testing system for Kilotest, informed by how QAI is structured. I read the QAI source (`src/util.test.ts`, `src/requestHandler.test.ts`, `package.json`) and the relevant parts of Kilotest (`api/*.js`, root `util.js`, `db/`, `web/listReports`) to ground this in what actually exists today, rather than in the abstract.
+
+### What QAI does that Kilotest should converge toward
+
+1. Colocated `*.test.ts` files next to the modules they test, run with the built-in `node:test` runner and `node:assert/strict`, no third-party test framework or mocking library.
+2. Coverage enforced by `c8` with 100% thresholds (lines/functions/branches/statements) as a gate, wired into `npm run check` alongside `typecheck`, `lint`, and `hint`.
+3. Server-level tests spin up a real ephemeral `http.Server` per test (`server.listen(0, ...)`) using `makeHandler(commentsFilePath, sendAlert)`, i.e. the handler factory takes its data-file path and side-effecting collaborators (like the alert sender) as parameters instead of importing hardcoded module-level constants. That is what makes deterministic, isolated tests possible: each test gets its own temp file/dir (`mkdtempDisposable`) and its own server instance, so there is no shared mutable fixture and no test order dependency.
+4. HTML responses are asserted with `node-html-parser`, not regex-only, so tests check structure (`querySelector('title')`, absence of `<script>`/`<img>` after sanitization) as well as text content.
+5. Failure paths are tested deliberately, not incidentally: unreadable/unwritable files (via `chmod`), missing fields, oversized/undersized input, duplicate detection, a broken `sendAlert` collaborator, etc. Every branch has a dedicated test, which is what makes 100% coverage meaningful rather than a vanity number.
+6. `node:test`'s built-in `testContext.mock.property(...)` is used for surgical monkey-patching (e.g. redirecting a route's file path) instead of a mocking library.
+
+### Why Kilotest's current setup cannot get there yet
+
+- `db/`, `db/reports`, `db/jobs`, etc. are computed once at module load as `path.join(__dirname, 'db')` in the top-level `util.js` (`dbPath`, `reportsPath`, `hiddenReportsPath`, `reportsExtractPath`, `jobsPath`, `recsPath`). Every downstream function (`getReportExtracts`, `getReportBasics`, job-claiming code, etc.) closes over these constants. There is currently no way to point any of this at a fixture directory the way QAI's `makeHandler(commentsFilePath, ...)` accepts a path.
+- `api/testSuite.js` is an end-to-end script, not a test suite: it talks to a live running server (local or deployed), picks a **random real report** (`Math.floor(reportsBasics.length * Math.random())`), and only asserts loose shape/presence conditions (`!issuesBasics.length`, `!issueBasics['impact on a user']`) because it cannot know in advance what the "correct" content of a real report is. It cannot catch a case where, say, `outcome === 'error'` is silently mapped to the wrong impact label, because it never encodes what the right answer should be.
+- There is no `test/` or `*.test.js` infrastructure, no `c8`, and no UI test of any kind.
+
+That correctly diagnoses the deficiency: you cannot write an assertion like "a report whose act has `outcome: 'error'` must be classified as X" against a randomly-selected real report, because you would first have to reverse-engineer what X should be from the code you are trying to test — that is circular.
+
+### Proposed plan
+
+#### Phase 0: Establish the toolchain
+- Add `c8` as a dev dependency; add an `npm test` script (`c8 --check-coverage node --test`) that actually runs something, replacing the current `"echo \"Error…\""` placeholder. Keep `test:api` for now as a separate, explicitly-labeled smoke/E2E script (see Phase 4) rather than deleting it immediately.
+- Decide coverage thresholds up front (QAI uses 100%; Kilotest may want to start lower, e.g. 80%, and ratchet up, given the amount of legacy code to backfill). This is a judgment call worth confirming with you before locking in a number in `package.json`.
+- Since Kilotest is CommonJS (`sourceType: "commonjs"` in `eslint.config.mjs`) and plain JS (not TypeScript), tests will be `*.test.js` using `require('node:test')` / `require('node:assert/strict')` — same runner and assertion library as QAI, just without the TS layer. This keeps the eventual TS migration (if you pursue it) a mechanical follow-up rather than a rewrite.
+
+#### Phase 1: Make data locations injectable (prerequisite for everything else)
+This is the load-bearing architectural change. Without it, no module can be tested against crafted fixtures without touching the real `db/`.
+- Refactor root `util.js` (and anything relying on its exported path constants) so the base data directory is resolved from a parameter or an environment variable read at call time (e.g. `KILOTEST_DB_DIR`, defaulting to today's `path.join(__dirname, 'db')`), rather than baked into module-load-time constants.
+- Where feasible, prefer explicit parameters over env vars for the innermost functions (mirroring `makeHandler(commentsFilePath, ...)`), and only use an env var at the outermost/server-bootstrap layer, so unit tests can call functions directly with a fixture path and never need to touch process env or the module cache.
+
+#### Phase 2: Build a crafted fixture corpus
+- Create `test/fixtures/db/` (or similar) with hand-authored, minimal Testaro reports covering the `outcome` cases you need verified: `success`, `failure`, `error`, mixed-outcome act arrays, an act with a missing/undefined `outcome`, a superseded report, an empty-violations report, etc.
+- Each fixture should be small enough that the expected API response can be hand-computed and hard-coded into the test as the "known-good" answer, exactly the property the current `test:api` script lacks.
+
+#### Phase 3: Unit tests, module by module
+Priority order, since these are the modules you just changed and where correctness of `outcome` handling matters most:
+1. `api/util.js` and root `util.js` (pure/near-pure helper functions: `getReportBasics`, `getReportExtracts`, `getToolsFacts`, response-metadata helpers).
+2. `api/listIssues.js`, `api/listViolators.js`, `api/listDiagnoses.js` (the modules that already reference `outcome`).
+3. `api/listReports.js`, `api/getReport.js`.
+4. `api/requestTest.js`, `api/requestRetest.js`, `api/requestFeature.js`, `api/schemas.js` (input validation edge cases: too-short/long fields, malformed bodies).
+Each test file lives next to its module (`api/listIssues.test.js`) and exercises it directly against Phase 2 fixtures, no server involved.
+
+#### Phase 4: Rework the API-level integration test
+Replace `test:api`'s "start from a live server, pick a random real report" approach with a QAI-style handler test:
+- Start an ephemeral server per test (or a shared one where safe) pointed at the fixture `db/` from Phase 2.
+- Walk through the same request sequence `testSuite.js` currently performs, but with hard-coded fixture identifiers and asserted exact expected values instead of "does this look non-empty."
+- Keep a much smaller, separate "smoke test against a real running deployment" script if you still want a pre-deploy sanity check, but do not conflate it with correctness testing.
+
+#### Phase 5: UI/web testing
+Kilotest's `web/*` pages are plain HTML + vanilla JS (no framework), analogous to QAI's server-rendered pages. Two complementary layers, matching QAI's toolset:
+- **Static/structural**: use `node-html-parser` (add as a dev dependency, same as QAI) to assert on the HTML served for each page/form (titles, presence of expected form fields, absence of raw unescaped user content).
+- **Behavioral**: for pages with client-side JS (`web/listReports/index.js` etc.) that fetch from the API and render results, either (a) extract the rendering logic into pure, testable functions that take API response JSON and return DOM-ready data/markup, so they can be unit-tested with `node:test` + `node:assert` without a browser, and/or (b) introduce a small number of real-browser tests (e.g. Playwright) only for the handful of interactions that cannot be verified any other way (form submission, client-side validation feedback). I would only reach for a browser automation tool if pure-function extraction proves impractical, since it adds real complexity and flakiness risk to a JS-only project.
+
+#### Phase 6: CI wiring and ratcheting
+- Add the test/coverage run to `.github/workflows/` (there is already a `.github/workflows/` dir) so `npm test` (and `npm run lint`) gate merges.
+- Track coverage percentage over time and raise the `c8` threshold as backfill work completes, rather than requiring 100% from day one across legacy code.
+
+#### On architectural convergence with QAI more broadly
+Beyond testing, actually adopting the QAI architecture would additionally involve: TypeScript with `--strict`, ESM (`"type": "module"`), the `@stylistic/eslint-plugin` + `@typescript-eslint/parser` lint stack, and the dependency-injection pattern (`makeHandler(path, collaborators)`) throughout, not just at the edges touched for testing. I would treat that as a separate, larger migration decided after the testing foundation above exists, since attempting both at once (rewrite the language/module system *and* build the test suite that is supposed to protect the rewrite) inverts the safety order.
+
+I have not made any changes yet. Given the phased nature of this, I would suggest reviewing/adjusting each phase before I implement it, starting with Phase 1 (the data-path injection refactor), since every later phase depends on it.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrpool/kilotest",
-  "version": "45.0.4",
+  "version": "46.0.0",
   "description": "Ensemble testing for front-end web quality",
   "main": "index.js",
   "engines": {

package/util.js

@@ -560,9 +560,9 @@ exports.getReportData = async (timeStamp, jobID) => {
       const instances = result?.standardResult?.instances ?? [];
       // For each standard instance of the act:
       instances.forEach(instance => {
-        const {catalogIndex, issueID} = instance;
-        // If it has a non-ignorable classified issue ID:
-        if (issueID && issueSpecs[issueID] && issueID !== 'ignorable') {
+        const {catalogIndex, issueID, outcome} = instance;
+        // If it reports a violation and has a non-ignorable classified issue ID:
+        if (outcome !== 'cantTell' && issueID && issueSpecs[issueID] && issueID !== 'ignorable') {
           // Ensure that the rule engine is in the temporary data.
           reporterIDSet.add(which);
           // Ensure that the issue is in the temporary data.