@jrpool/kilotest 46.0.8 → 46.0.14

package/web/requestTest/requestTest.test.js

@@ -0,0 +1,115 @@
+/*
+  requestTest.test.js
+  Unit tests for web/requestTest/index.js, covering the success path of answer.
+*/
+
+// IMPORTS
+
+const {test, before, after, beforeEach} = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const {parse} = require('node-html-parser');
+
+// CONSTANTS
+
+const fixtureDBDir = path.join(__dirname, '..', '..', 'test', 'fixtures', 'db');
+const recsPath = path.join(fixtureDBDir, 'jobs', 'recs.json');
+
+// SETUP AND TEARDOWN
+
+before(async () => {
+  process.env.DB_DIR = fixtureDBDir;
+  process.env.AUTH_CODE = 'test-auth-code';
+});
+
+beforeEach(async () => {
+  // Reset recs.json and clean job directories before each test.
+  await fs.writeFile(recsPath, '{}\n');
+  for (const sub of ['claimed', 'queue', 'failed']) {
+    const dir = path.join(fixtureDBDir, 'jobs', sub);
+    await fs.mkdir(dir, {recursive: true});
+    const files = await fs.readdir(dir).catch(() => []);
+    for (const file of files) {
+      try {
+        await fs.unlink(path.join(dir, file));
+      }
+      catch {
+        // Ignore cleanup errors for files that may not exist.
+      }
+    }
+  }
+});
+
+after(async () => {
+  // Restore recs.json and clean job directories after all tests.
+  await fs.writeFile(recsPath, '{}\n');
+  for (const sub of ['claimed', 'queue', 'failed']) {
+    const dir = path.join(fixtureDBDir, 'jobs', sub);
+    await fs.mkdir(dir, {recursive: true});
+    const files = await fs.readdir(dir).catch(() => []);
+    for (const file of files) {
+      try {
+        await fs.unlink(path.join(dir, file));
+      }
+      catch {
+        // Ignore cleanup errors for files that may not exist.
+      }
+    }
+  }
+});
+
+// TESTS
+
+test('answer returns ok with a populated answer page for a valid test request', async () => {
+  // Require the module after DB_DIR is set.
+  const {answer} = require('./index');
+  const result = await answer(
+    'Test Page', 'https://example.com/test-success', 'Because accessibility'
+  );
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage, 'answerPage should be present');
+  // Verify the placeholders were replaced.
+  assert.ok(result.answerPage.includes('Test Page'));
+  assert.ok(result.answerPage.includes('Because accessibility'));
+  assert.ok(!result.answerPage.includes('__target__'));
+  assert.ok(!result.answerPage.includes('__why__'));
+  // Verify the HTML is well-formed.
+  const doc = parse(result.answerPage);
+  const h1 = doc.querySelector('h1');
+  assert.ok(h1);
+  assert.ok(h1.textContent.includes('Test Page'));
+});
+
+test('answer returns an error when the URL is already queued', async () => {
+  const {answer} = require('./index');
+  // Create a queued job with the target URL.
+  const queueDir = path.join(fixtureDBDir, 'jobs', 'queue');
+  const jobPath = path.join(queueDir, 'queuedJob.json');
+  await fs.writeFile(jobPath, JSON.stringify({
+    id: 'queuedJob',
+    target: {what: 'Queued Page', url: 'https://example.com/queued'}
+  }));
+  try {
+    const result = await answer(
+      'Queued Page', 'https://example.com/queued', 'Because accessibility'
+    );
+    assert.equal(result.status, 'error');
+    assert.ok(result.message.includes('queued'));
+  }
+  finally {
+    try {
+      await fs.unlink(jobPath);
+    }
+    catch {
+      // Ignore cleanup errors.
+    }
+  }
+});
+
+test('answer returns an error for an invalid recommendation', async () => {
+  const {answer} = require('./index');
+  const result = await answer('', 'not-a-url', 'why');
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'Invalid recommendation');
+});

package/web/requestTestForm/requestTestForm.test.js

@@ -0,0 +1,40 @@
+/*
+  requestTestForm.test.js
+  Unit tests for web/requestTestForm/index.js.
+*/
+
+// IMPORTS
+
+const {test} = require('node:test');
+const assert = require('node:assert/strict');
+const {parse} = require('node-html-parser');
+const {answer} = require('./index');
+
+// TESTS
+
+test('requestTestForm returns an ok status with valid HTML', async () => {
+  const result = await answer();
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage);
+  const html = parse(result.answerPage);
+  assert.equal(html.querySelector('title').text, 'Test recommendation | Kilotest');
+});
+
+test('requestTestForm includes a form that posts to /requestTest.html', async () => {
+  const result = await answer();
+  const html = parse(result.answerPage);
+  const form = html.querySelector('form');
+  assert.ok(form);
+  assert.equal(form.getAttribute('action'), '/requestTest.html');
+  assert.equal(form.getAttribute('method'), 'post');
+});
+
+test('requestTestForm includes required inputs for what, url, and why', async () => {
+  const result = await answer();
+  const html = parse(result.answerPage);
+  const inputs = html.querySelectorAll('input');
+  const names = inputs.map(input => input.getAttribute('name'));
+  assert.ok(names.includes('what'));
+  assert.ok(names.includes('url'));
+  assert.ok(names.includes('why'));
+});

package/web/rewindReportsForm/rewindReportsForm.test.js

@@ -0,0 +1,86 @@
+/*
+  rewindReportsForm.test.js
+  Unit tests for web/rewindReportsForm/index.js.
+*/
+
+// IMPORTS
+
+const {test, before, after} = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const fs = require('fs/promises');
+const {parse} = require('node-html-parser');
+const {answer} = require('./index');
+const {reportsPath} = require('../../util');
+
+// SETUP AND TEARDOWN
+
+const savedAuthCode = process.env.AUTH_CODE;
+const savedDBDir = process.env.DB_DIR;
+const fixtureDBDir = path.join(__dirname, '..', '..', 'test', 'fixtures', 'db');
+
+before(() => {
+  process.env.AUTH_CODE = 'test-auth-code';
+  process.env.DB_DIR = fixtureDBDir;
+});
+
+after(async () => {
+  if (savedAuthCode !== undefined) {
+    process.env.AUTH_CODE = savedAuthCode;
+  }
+  if (savedDBDir !== undefined) {
+    process.env.DB_DIR = savedDBDir;
+  }
+});
+
+// TESTS
+
+test('rewindReportsForm displays a list of reports when no submission', async () => {
+  const result = await answer(null, '');
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage);
+  const html = parse(result.answerPage);
+  assert.ok(html.querySelector('title'));
+});
+
+test('rewindReportsForm returns an error for an invalid auth code', async () => {
+  const result = await answer(null, 'authCode=wrong&report=260202T0000-new');
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'Invalid authorization code');
+});
+
+test('rewindReportsForm returns an error when deleting a nonexistent report', async () => {
+  const result = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
+  assert.equal(result.status, 'error');
+  assert.ok(result.message.includes('Deleting latest superseding reports'));
+});
+
+test('rewindReportsForm deletes a latest superseding report with valid auth code', async () => {
+  // 260202T0000-new is the latest report for its URL, superseding 260101T0000-mix.
+  const reportPath = path.join(reportsPath(), '260202T0000-new.json');
+  const backup = await fs.readFile(reportPath, 'utf8');
+  try {
+    const result = await answer(null, 'authCode=test-auth-code&report=260202T0000-new');
+    assert.equal(result.status, 'ok');
+    await fs.access(reportPath).then(
+      () => {throw new Error('Report should have been deleted');},
+      () => {}
+    );
+  }
+  finally {
+    await fs.writeFile(reportPath, backup);
+  }
+});
+
+test('rewindReportsForm returns an error when a report file is corrupt', async () => {
+  const reportPath = path.join(reportsPath(), '260101T0001-ct.json');
+  const backup = await fs.readFile(reportPath, 'utf8');
+  try {
+    await fs.writeFile(reportPath, 'not valid json');
+    const result = await answer(null, '');
+    assert.equal(result.status, 'error');
+  }
+  finally {
+    await fs.writeFile(reportPath, backup);
+  }
+});

package/web/tutorial/index.html

@@ -334,7 +334,7 @@
         <section id="testing-page">
           <h3>Testing the page</h3>
           <h4>By rule engines</h4>
-          <p>What do rule engines find about this issue when they test this page? For a quick answer, Kilotest subjected the page to testing by all 10 rule engines in its ensemble. Two of the rule engines reported rule violations belonging to this issue:</p>
+          <p>What do rule engines find about this issue when they test this page? For a quick answer, Kilotest subjected the page to testing by the 10 rule engines that were then in its ensemble (since then, 2 more were added, so now there are 12). Two of the rule engines reported rule violations belonging to this issue:</p>
           <ul>
             <li>The Testaro rule engine reported, <q>input has no autocomplete="email" attribute</q>. In other words, Testaro recognized this input as asking for the user&rsquo;s email address and therefore as requiring an <code>autocomplete="email"</code> attribute, but not having one.</li>
             <li>The HTML CodeSniffer rule engine, referencing this <code>form</code> element, reported, <q>Invalid autocomplete value: new-password. Element does not belong to Password control group.</q>. In other words, HTML CodeSniffer recognized that this form is not a password input (a form is not an input at all!) and therefore is not eligible for an <code>autocomplete="new-password"</code> attribute, but has one.</li>
@@ -346,7 +346,7 @@
           </ul>
           <p>You might think that this shows rule engines were effective in revealing the defects related to this issue, but this is not a success story. Yes, two rule engines found related defects, but the other eight rule engines did not. Inspection of those eight rule engines reveals that six of them include rules enforcing this standard in some way. So, did these six rule engines report false negatives? Did they fail to enforce their applicable rules here? Perhaps, but perhaps not. Suppose a rule engine has a rule that requires the value of any <code>autocomplete</code> attribute to be in the list of allowed values. Well, <code>new-password</code> is in the list. So that rule engine would, properly, not report a violation of <strong>that</strong> rule.</p>
           <p>We can clearly see from this example that no rule engine will find all accessibility defects, not even all defects that <strong>can</strong> be detected by rule engines. That is why Kilotest uses an <strong>ensemble</strong> of rule engines.</p>
-          <p>Of course, running ten rule engines also creates a risk of false positives reported by ten rule engines. In this particular case, there were none. For example, no rule engine claimed to find a defect related to this issue someplace else on the page. Kilotest, like any platform that integrates multiple rule engines, faces greater risks of false positives than a single rule engine would. It combats false positives by investigating user complaints and deprecating rules whose tests are found faulty.</p>
+          <p>Of course, running twelve rule engines also creates a risk of false positives reported by twelve rule engines. In this particular case, there were none. For example, no rule engine claimed to find a defect related to this issue someplace else on the page. Kilotest, like any platform that integrates multiple rule engines, faces greater risks of false positives than a single rule engine would. It combats false positives by investigating user complaints and deprecating rules whose tests are found faulty.</p>
           <h4>By AI agents</h4>
           <p>An AI agent powered by a frontier model was asked to visit the page and report any accessibility defect related to <code>autocomplete</code> attributes. The agent worked for about 3 minutes and then reported a missing <code>autocomplete="email"</code> attribute in the newsletter subscription form. But the page has no newsletter subscription form! It has a form to request a guide, not to subscribe to anything. Moreover, the agent provided code to support its conclusion, and the code differed pervasively from the actual code on the page. This response was a complete fabrication. The agent confessed to it, explaining that it resorted to hallucination when its security restrictions prevented it from accessing the necessary page data.</p>
           <p>A similar AI agent returned correct diagnoses when it was fed the page as an HTML document. That worked, because the two defects relevant to this practicum are discoverable from the HTML of the rendered page. Any defects arising from stylesheets or scripts would have been missed.</p>
@@ -418,7 +418,7 @@
               <legend>Choose the best answer</legend>
               <p>In this practicum, which rule engines discovered both <code>autocomplete</code> defects?</p>
               <ol>
-                <li><label><input type="radio" name="q4" value="a"> All 10 rule engines</label></li>
+                <li><label><input type="radio" name="q4" value="a"> All 12 rule engines</label></li>
                 <li><label><input type="radio" name="q4" value="b"> Six of the rule engines</label></li>
                 <li><label><input type="radio" name="q4" value="c"> Only Testaro and HTML CodeSniffer</label></li>
                 <li><label><input type="radio" name="q4" value="d"> None of the rule engines</label></li>

package/web/tutorial/tutorial.test.js

@@ -0,0 +1,97 @@
+/*
+  tutorial.test.js
+  Unit tests for web/tutorial/index.js, covering the answer and handleComment exports.
+*/
+
+// IMPORTS
+
+const {test, before, after} = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const {answer, handleComment} = require('./index');
+
+// CONSTANTS
+
+const commentsPath = path.join(__dirname, 'comments.json');
+
+// SETUP AND TEARDOWN
+
+let originalComments;
+
+before(async () => {
+  originalComments = await fs.readFile(commentsPath, 'utf8').catch(() => null);
+});
+
+after(async () => {
+  if (originalComments !== null) {
+    await fs.writeFile(commentsPath, originalComments);
+  }
+  else {
+    await fs.unlink(commentsPath).catch(() => {});
+  }
+});
+
+// TESTS
+
+test('answer returns the tutorial page with status ok', async () => {
+  const result = await answer();
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage);
+  assert.ok(result.answerPage.length > 0);
+});
+
+test('handleComment returns an error for empty content', async () => {
+  const result = await handleComment('');
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'No content provided');
+});
+
+test('handleComment returns an error for non-string content', async () => {
+  const result = await handleComment(null);
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'No content provided');
+});
+
+test('handleComment returns an error for content that is empty after sanitization', async () => {
+  const result = await handleComment('<script></script>');
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'Comment is empty after sanitization');
+});
+
+test('handleComment saves a sanitized comment and returns ok', async () => {
+  // Replace comments with an empty array for a clean test.
+  await fs.writeFile(commentsPath, '[]\n');
+  const result = await handleComment('This is a <b>test</b> comment.');
+  assert.equal(result.status, 'ok');
+  const comments = JSON.parse(await fs.readFile(commentsPath, 'utf8'));
+  assert.equal(comments.length, 1);
+  assert.equal(comments[0].content, 'This is a test comment.');
+  assert.ok(comments[0].timeStamp);
+});
+
+test('handleComment strips HTML tags and control characters', async () => {
+  await fs.writeFile(commentsPath, '[]\n');
+  const result = await handleComment('<img src=x onerror=alert(1)>\x00\x07Hello');
+  assert.equal(result.status, 'ok');
+  const comments = JSON.parse(await fs.readFile(commentsPath, 'utf8'));
+  assert.equal(comments[0].content, 'Hello');
+});
+
+test('handleComment truncates content to 500 characters', async () => {
+  await fs.writeFile(commentsPath, '[]\n');
+  const longComment = 'x'.repeat(600);
+  const result = await handleComment(longComment);
+  assert.equal(result.status, 'ok');
+  const comments = JSON.parse(await fs.readFile(commentsPath, 'utf8'));
+  assert.equal(comments[0].content.length, 500);
+});
+
+test('handleComment creates comments.json when it does not exist', async () => {
+  await fs.unlink(commentsPath).catch(() => {});
+  const result = await handleComment('Test comment for missing file');
+  assert.equal(result.status, 'ok');
+  const comments = JSON.parse(await fs.readFile(commentsPath, 'utf8'));
+  assert.equal(comments.length, 1);
+  assert.equal(comments[0].content, 'Test comment for missing file');
+});

package/web/unhideReportForm/unhideReportForm.test.js

@@ -0,0 +1,135 @@
+/*
+  unhideReportForm.test.js
+  Unit tests for web/unhideReportForm/index.js.
+*/
+
+// IMPORTS
+
+const {test, before, after, beforeEach} = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const fs = require('fs/promises');
+const {parse} = require('node-html-parser');
+const {answer} = require('./index');
+const {reportsPath, hiddenReportsPath} = require('../../util');
+
+// CONSTANTS
+
+const hiddenFile = '260101T0007-hid.json';
+
+// SETUP AND TEARDOWN
+
+const savedAuthCode = process.env.AUTH_CODE;
+const savedDBDir = process.env.DB_DIR;
+const fixtureDBDir = path.join(__dirname, '..', '..', 'test', 'fixtures', 'db');
+
+const getPaths = () => ({
+  hidden: path.join(hiddenReportsPath(), hiddenFile),
+  report: path.join(reportsPath(), hiddenFile)
+});
+
+before(() => {
+  process.env.AUTH_CODE = 'test-auth-code';
+  process.env.DB_DIR = fixtureDBDir;
+});
+
+beforeEach(async () => {
+  // Ensure the hidden report is in the hidden directory before each test.
+  const {hidden, report} = getPaths();
+  try {
+    await fs.rename(report, hidden);
+  }
+  catch {
+    // Report may already be hidden.
+  }
+});
+
+after(async () => {
+  // Restore the hidden report if it was moved.
+  const {hidden, report} = getPaths();
+  try {
+    await fs.rename(report, hidden);
+  }
+  catch {
+    // Report may already be hidden.
+  }
+  if (savedAuthCode !== undefined) {
+    process.env.AUTH_CODE = savedAuthCode;
+  }
+  if (savedDBDir !== undefined) {
+    process.env.DB_DIR = savedDBDir;
+  }
+});
+
+// TESTS
+
+test('unhideReportForm displays a list of hidden reports when no submission', async () => {
+  const result = await answer(null, '');
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage);
+  const html = parse(result.answerPage);
+  const radios = html.querySelectorAll('input[type="radio"]');
+  assert.ok(radios.length > 0);
+});
+
+test('unhideReportForm returns an error for an invalid auth code', async () => {
+  const result = await answer(null, 'authCode=wrong&report=260101T0007-hid');
+  assert.equal(result.status, 'error');
+  assert.equal(result.message, 'Invalid authorization code');
+});
+
+test('unhideReportForm unhides a report with valid auth code', async () => {
+  const {report} = getPaths();
+  const result = await answer(null, 'authCode=test-auth-code&report=260101T0007-hid');
+  assert.equal(result.status, 'ok');
+  // The report should now be in the reports directory.
+  await fs.access(report);
+});
+
+test('unhideReportForm returns an error when unhiding a nonexistent report', async () => {
+  const result = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
+  assert.equal(result.status, 'error');
+  assert.ok(result.message.includes('Unhiding report'));
+});
+
+test('unhideReportForm sorts hidden reports with the same page name by timeStamp', async () => {
+  const {hidden} = getPaths();
+  // Add a second hidden report with the same what as the existing one.
+  const existing = JSON.parse(await fs.readFile(hidden, 'utf8'));
+  // Create a copy with a different timeStamp but same target.what.
+  const copy = JSON.parse(JSON.stringify(existing));
+  copy.id = '260101T0010-hid2';
+  const copyPath = path.join(hiddenReportsPath(), '260101T0010-hid2.json');
+  await fs.writeFile(copyPath, JSON.stringify(copy));
+  try {
+    const result = await answer(null, '');
+    assert.equal(result.status, 'ok');
+    // Both reports should appear in the form.
+    assert.ok(result.answerPage.includes('260101T0007-hid'));
+    assert.ok(result.answerPage.includes('260101T0010-hid2'));
+  }
+  finally {
+    await fs.unlink(copyPath).catch(() => {});
+  }
+});
+
+test('unhideReportForm sorts hidden reports with different page names alphabetically', async () => {
+  const {hidden} = getPaths();
+  // Add a second hidden report with a different what.
+  const existing = JSON.parse(await fs.readFile(hidden, 'utf8'));
+  const copy = JSON.parse(JSON.stringify(existing));
+  copy.id = '260101T0011-hid3';
+  copy.target.what = 'A Different Page';
+  const copyPath = path.join(hiddenReportsPath(), '260101T0011-hid3.json');
+  await fs.writeFile(copyPath, JSON.stringify(copy));
+  try {
+    const result = await answer(null, '');
+    assert.equal(result.status, 'ok');
+    // Both reports should appear in the form.
+    assert.ok(result.answerPage.includes('260101T0007-hid'));
+    assert.ok(result.answerPage.includes('260101T0011-hid3'));
+  }
+  finally {
+    await fs.unlink(copyPath).catch(() => {});
+  }
+});