@jrpool/kilotest 48.0.0 → 50.0.0

package/web/ai0BalanceForm/index.html

@@ -15,7 +15,7 @@
       <h1><a href="/">Kilotest</a>: Record AI service 0 balance</h1>
       <p>__oldBalance__ recorded AI service 0 balance.</p>
       <p>The current AI service 0 balance is available on the <a href="https://platform.claude.com/settings/billing">Claude Platform Billing page</a>.</p>
-      <form action="/ai0BalanceForm.html">
+      <form action="/ai0BalanceForm.html" method="post">
         <p><label>
           Balance (in $):
           <input type="number" size="5" min="0" max="99.99" step="0.01" name="newBalance" required>

package/web/ai0BalanceForm/index.ts

@@ -7,20 +7,22 @@
 
 import fs from 'node:fs/promises';
 import {balancePath} from '../../balances.ts';
-import {getJSON, populateTemplate} from '../../util.ts';
+import {getJSON, isValidAuthCode, populateTemplate} from '../../util.ts';
 
 // FUNCTIONS
 
-// Returns a form for recording the AI service 0 balance.
-export const answer = async (_: any, search: string) => {
+// Returns a form for recording the AI service 0 balance. A GET request never processes a
+// submission, regardless of its query string; only a POST request (the form's own
+// submission) does.
+export const answer = async (_: any, search: string, method: string) => {
   const searchParams = new URLSearchParams(search);
   const authCode = searchParams?.get('authCode');
   const newBalanceString = searchParams?.get('newBalance');
   let oldBalance: any;
-  // If the form displayed itself:
-  if (newBalanceString) {
+  // If the form has been submitted:
+  if (method === 'POST' && newBalanceString) {
     // If the authorization code is valid:
-    if (authCode === process.env.AUTH_CODE) {
+    if (isValidAuthCode(authCode)) {
       const newBalance = Number.parseFloat(newBalanceString);
       // If the new balance is valid:
       if (
@@ -40,10 +42,12 @@ export const answer = async (_: any, search: string) => {
     }
     // Otherwise, i.e. if the authorization code is invalid:
     else {
-      // Report the error.
+      // Report the error, deliberately vague so as not to confirm to an attacker that the
+      // authorization code specifically (as opposed to some other part of the request) is
+      // what was wrong.
       return {
         status: 'error',
-        message: 'Invalid authorization code'
+        message: 'Invalid request'
       }
     }
   }

package/web/enqueue/index.ts

@@ -6,51 +6,45 @@
 // IMPORTS
 
 import {
-  deleteTestRequests, getJSON, getNowStamp, getRandomString, isURL, jobsPath, populateTemplate
+  deleteTestRequests, getJSON, getNowStamp, getRandomString, jobsPath, populateTemplate
 } from '../../util.ts';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
 // FUNCTIONS
 
-// Implements a test request approval and returns a revised request page.
-export const answer = async (url: string, description: string, authCode: string) => {
-  // If the arguments are valid:
-  if (isURL(url) && description && authCode === process.env.AUTH_CODE) {
-    // Get the job template.
-    const jobTemplateJSON = await fs.readFile(path.join(import.meta.dirname, '..', '..', 'job.json'), 'utf8');
-    const job = JSON.parse(jobTemplateJSON);
-    const nowStamp = getNowStamp();
-    // Populate the template with job properties.
-    const jobIDSuffix = getRandomString(3);
-    const jobName = `${nowStamp}-${jobIDSuffix}`;
-    job.id = jobName;
-    job.creationTimeStamp = nowStamp;
-    job.executionTimeStamp = nowStamp;
-    job.target.what = description;
-    job.target.url = url;
-    const query: Record<string, string> = {
-      target: description,
-      jobName
-    };
-    // Save the job in the queue.
-    await fs.writeFile(
-      path.join(jobsPath(), 'queue', `${jobName}.json`), getJSON(job)
-    );
-    console.log(`Retest queued for ${description} as job ${jobName}`);
-    // Delete the test requests for the target.
-    await deleteTestRequests(url);
-    // Get the populated answer template.
-    const answerPage = await populateTemplate(import.meta.dirname, query);
-    // Return the populated page.
-    return {
-      status: 'ok',
-      answerPage
-    };
-  }
-  // Otherwise, i.e. if the authorization code is invalid, return an error page.
+// Implements a test request approval and returns a revised request page. The caller
+// (index.ts, this function's only caller) has already validated the URL and
+// authorization code before invoking this function.
+export const answer = async (url: string, description: string) => {
+  // Get the job template.
+  const jobTemplateJSON = await fs.readFile(path.join(import.meta.dirname, '..', '..', 'job.json'), 'utf8');
+  const job = JSON.parse(jobTemplateJSON);
+  const nowStamp = getNowStamp();
+  // Populate the template with job properties.
+  const jobIDSuffix = getRandomString(3);
+  const jobName = `${nowStamp}-${jobIDSuffix}`;
+  job.id = jobName;
+  job.creationTimeStamp = nowStamp;
+  job.executionTimeStamp = nowStamp;
+  job.target.what = description;
+  job.target.url = url;
+  const query: Record<string, string> = {
+    target: description,
+    jobName
+  };
+  // Save the job in the queue.
+  await fs.writeFile(
+    path.join(jobsPath(), 'queue', `${jobName}.json`), getJSON(job)
+  );
+  console.log(`Retest queued for ${description} as job ${jobName}`);
+  // Delete the test requests for the target.
+  await deleteTestRequests(url);
+  // Get the populated answer template.
+  const answerPage = await populateTemplate(import.meta.dirname, query);
+  // Return the populated page.
   return {
-    status: 'error',
-    message: 'Invalid authorization code'
+    status: 'ok',
+    answerPage
   };
 };

package/web/expungeReportsForm/expungeReportsForm.test.ts

@@ -35,32 +35,40 @@ after(async () => {
 
 // TESTS
 
-test('expungeReportsForm displays a list of reports when no submission', async () => {
-  const result: any = await answer(null, '');
+test('expungeReportsForm displays a list of reports on a GET request', async () => {
+  const result: any = await answer(null, '', 'GET');
   assert.equal(result.status, 'ok');
   assert.ok(result.answerPage);
   const html = parse(result.answerPage);
   assert.ok(html.querySelector('title'));
 });
 
-test('expungeReportsForm returns an error for an invalid auth code', async () => {
-  const result: any = await answer(null, 'authCode=wrong&report=260101T0001-ct');
+test('expungeReportsForm ignores a report query string on a GET request', async () => {
+  const reportPath = path.join(reportsPath(), '260101T0001-ct.json');
+  const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct', 'GET');
+  assert.equal(result.status, 'ok');
+  // The report should NOT have been deleted by a mere GET request.
+  await fs.access(reportPath);
+});
+
+test('expungeReportsForm returns an error for an invalid auth code on a POST request', async () => {
+  const result: any = await answer(null, 'authCode=wrong&report=260101T0001-ct', 'POST');
   assert.equal(result.status, 'error');
-  assert.equal(result.message, 'Invalid authorization code');
+  assert.equal(result.message, 'Invalid request');
 });
 
-test('expungeReportsForm returns an error when deleting a nonexistent report', async () => {
-  const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
+test('expungeReportsForm returns an error when deleting a nonexistent report on a POST request', async () => {
+  const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope', 'POST');
   assert.equal(result.status, 'error');
   assert.ok(result.message.includes('Deleting sole reports'));
 });
 
-test('expungeReportsForm deletes a sole report with valid auth code', async () => {
+test('expungeReportsForm deletes a sole report with valid auth code on a POST request', async () => {
   // 260101T0001-ct is the only report for its URL (canttell).
   const reportPath = path.join(reportsPath(), '260101T0001-ct.json');
   const backup = await fs.readFile(reportPath, 'utf8');
   try {
-    const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct');
+    const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct', 'POST');
     assert.equal(result.status, 'ok');
     await fs.access(reportPath).then(
       () => {throw new Error('Report should have been deleted');},
@@ -77,7 +85,7 @@ test('expungeReportsForm returns an error when a report file is corrupt', async
   const backup = await fs.readFile(reportPath, 'utf8');
   try {
     await fs.writeFile(reportPath, 'not valid json');
-    const result: any = await answer(null, '');
+    const result: any = await answer(null, '', 'GET');
     assert.equal(result.status, 'error');
   }
   finally {
@@ -100,7 +108,7 @@ test('expungeReportsForm shows no-deletable message when every URL has at least
   process.env.DB_DIR = tmpDir;
   try {
     const {answer} = await import('./index.ts');
-    const result: any = await answer(null, '');
+    const result: any = await answer(null, '', 'GET');
     assert.equal(result.status, 'ok');
     assert.ok(result.answerPage.includes('no reports to delete'));
     assert.ok(result.answerPage.includes('disabled'));

package/web/expungeReportsForm/index.html

@@ -14,7 +14,7 @@
     <main>
       <h1><a href="/">Kilotest</a>: Expunge reports</h1>
       <p>__intro__</p>
-      <form action="/expungeReportsForm.html">
+      <form action="/expungeReportsForm.html" method="post">
         <fieldset>
           <legend>Reports</legend>
           __reports__

package/web/expungeReportsForm/index.ts

@@ -10,7 +10,7 @@ import {reportDeletionForm} from '../reportDeletion.ts';
 // FUNCTIONS
 
 // Returns a form for deleting sole reports.
-export const answer = async (_: any, search: string) => reportDeletionForm(import.meta.dirname, search, {
+export const answer = async (_: any, search: string, method: string) => reportDeletionForm(import.meta.dirname, search, method, {
   intro: 'Choose the sole reports to delete.',
   emptyIntro: 'Each target has at least 2 reports, so there are no reports to delete.',
   failurePrefix: 'Deleting sole reports failed',

package/web/hideReportForm/hideReportForm.test.ts

@@ -44,8 +44,8 @@ after(async () => {
 
 // TESTS
 
-test('hideReportForm displays a list of reports when no submission', async () => {
-  const result: any = await answer(null, '');
+test('hideReportForm displays a list of reports on a GET request', async () => {
+  const result: any = await answer(null, '', 'GET');
   assert.equal(result.status, 'ok');
   assert.ok(result.answerPage);
   const html = parse(result.answerPage);
@@ -53,13 +53,28 @@ test('hideReportForm displays a list of reports when no submission', async () =>
   assert.ok(radios.length > 0);
 });
 
-test('hideReportForm returns an error for an invalid auth code', async () => {
-  const result: any = await answer(null, 'authCode=wrong&report=260101T0009-brd');
+test('hideReportForm ignores a report query string on a GET request', async () => {
+  const reportPath = path.join(reportsPath(), '260101T0009-brd.json');
+  const hiddenPath = path.join(hiddenReportsPath(), '260101T0009-brd.json');
+  try {
+    await fs.rename(hiddenPath, reportPath);
+  }
+  catch {
+    // Report may already be in reports.
+  }
+  const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd', 'GET');
+  assert.equal(result.status, 'ok');
+  // The report should NOT have been hidden by a mere GET request.
+  await fs.access(reportPath);
+});
+
+test('hideReportForm returns an error for an invalid auth code on a POST request', async () => {
+  const result: any = await answer(null, 'authCode=wrong&report=260101T0009-brd', 'POST');
   assert.equal(result.status, 'error');
-  assert.equal(result.message, 'Invalid authorization code');
+  assert.equal(result.message, 'Invalid request');
 });
 
-test('hideReportForm hides a report with valid auth code', async () => {
+test('hideReportForm hides a report with valid auth code on a POST request', async () => {
   const reportPath = path.join(reportsPath(), '260101T0009-brd.json');
   const hiddenPath = path.join(hiddenReportsPath(), '260101T0009-brd.json');
   // Ensure the report exists in reports and not in hidden.
@@ -69,14 +84,14 @@ test('hideReportForm hides a report with valid auth code', async () => {
   catch {
     // Report may already be in reports.
   }
-  const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd');
+  const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd', 'POST');
   assert.equal(result.status, 'ok');
   // The report should now be in the hidden directory.
   await fs.access(hiddenPath);
 });
 
-test('hideReportForm returns an error when hiding a nonexistent report', async () => {
-  const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
+test('hideReportForm returns an error when hiding a nonexistent report on a POST request', async () => {
+  const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope', 'POST');
   assert.equal(result.status, 'error');
   assert.ok(result.message.includes('Hiding report'));
 });

package/web/hideReportForm/index.html

@@ -14,7 +14,7 @@
     <main>
       <h1><a href="/">Kilotest</a>: Classify report as experimental</h1>
       <p>Choose a report to classify as experimental.</p>
-      <form action="/hideReportForm.html">
+      <form action="/hideReportForm.html" method="post">
         <fieldset>
           <legend>Reports</legend>
           __reports__

package/web/hideReportForm/index.ts

@@ -5,21 +5,24 @@
 
 // IMPORTS
 
-import {getReportExtracts, hiddenReportsPath, populateTemplate, reportsPath} from '../../util.ts';
+import {
+  getReportExtracts, hiddenReportsPath, isValidAuthCode, populateTemplate, reportsPath
+} from '../../util.ts';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
 // FUNCTIONS
 
-// Returns a form for hiding a report.
-export const answer = async (_: any, search: string) => {
+// Returns a form for hiding a report. A GET request never processes a submission,
+// regardless of its query string; only a POST request (the form's own submission) does.
+export const answer = async (_: any, search: string, method: string) => {
   const searchParams = new URLSearchParams(search);
   const authCode = searchParams?.get('authCode');
   const jobName = searchParams?.get('report');
-  // If the form has been displayed by itself after a submission and a report is to be hidden:
-  if (jobName) {
+  // If the form has been submitted and a report is to be hidden:
+  if (method === 'POST' && jobName) {
     // If the authorization code is valid:
-    if (authCode === process.env.AUTH_CODE) {
+    if (isValidAuthCode(authCode)) {
       const fileName = `${jobName}.json`;
       try {
         // Move the specified report to the directory of hidden reports.
@@ -36,10 +39,12 @@ export const answer = async (_: any, search: string) => {
     }
     // Otherwise, i.e. if the authorization code is invalid:
     else {
-      // Report the error.
+      // Report the error, deliberately vague so as not to confirm to an attacker that the
+      // authorization code specifically (as opposed to some other part of the request) is
+      // what was wrong.
       return {
         status: 'error',
-        message: 'Invalid authorization code'
+        message: 'Invalid request'
       }
     }
   }

package/web/listTopIssues/index.ts

@@ -79,7 +79,7 @@ const getIssuesSummary = async () => {
       // Report this.
       console.log(`ERROR: Annotations obsolete for issue ${issueID}; reannotate`);
       // Notify a manager.
-      sendAlert('Annotations obsolete', `Annotations for issue ${issueID} obsolete; reannotate.`);
+      sendAlert('Kilotest: Annotations obsolete', `Annotations for issue ${issueID} obsolete; reannotate.`);
     }
   });
   // For each summarized issue:

package/web/manage/index.html

@@ -23,9 +23,10 @@
         <li><a href="/rewindReportsForm.html">Delete latest superseding reports</a></li>
         <li><a href="/expungeReportsForm.html">Delete sole reports</a></li>
         <li><a href="/hideReportForm.html">Classify a report as experimental</a></li>
-        <li><a href="/unhideReportForm.html">Declassify a report as experimental</a></li>
+        <li><a href="/showHiddenReportsForm.html">View the list of experimental reports</a></li>
         <li><a href="/ai0BalanceForm.html">Record the current balance of AI service 0</a></li>
         <li><a href="/renewWCAGForm.html">Renew the map of WCAG numeric to text identifiers</a></li>
+        <li><a href="/metrics.html">View usage metrics</a></li>
       </ul>
     </main>
   </body>

package/web/metrics/index.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="publisher" content="Jonathan Robert Pool">
+    <meta name="creator" content="Jonathan Robert Pool">
+    <meta name="keywords" content="report,accessibility,a11y">
+    <title>Usage metrics | Kilotest</title>
+    <link rel="icon" href="/favicon.ico">
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <main>
+      <h1><a href="/">Kilotest</a>: Usage metrics</h1>
+      __body__
+    </main>
+  </body>
+</html>

package/web/metrics/index.ts

@@ -0,0 +1,127 @@
+/*
+  index.ts
+  Serves usage metrics for the maintainer: web page views, MCP tool calls, and API
+  operation calls, each broken down by name, recorded since a given date and time.
+*/
+
+// IMPORTS
+
+import {
+  clearMetrics, getDateTimeString, getExclusionCookieValue, getMetrics, isValidAuthCode,
+  metricsExclusionCookieName, populateTemplate
+} from '../../util.ts';
+
+// FUNCTIONS
+
+// Returns an HTML table of the counts in a metrics category, sorted by descending count.
+const getCategoryTable = (counts: Record<string, number>): string => {
+  const margin = ' '.repeat(6);
+  const names = Object.keys(counts).sort((a, b) => counts[b]! - counts[a]!);
+  if (!names.length) {
+    return `${margin}<p>None yet.</p>`;
+  }
+  const rows = names.map(name => `${margin}    <tr><td>${name}</td><td>${counts[name]}</td></tr>`);
+  return [
+    `${margin}<table class="allBorder">`,
+    `${margin}  <thead>`,
+    `${margin}    <tr><th>Name</th><th>Count</th></tr>`,
+    `${margin}  </thead>`,
+    `${margin}  <tbody>`,
+    ...rows,
+    `${margin}  </tbody>`,
+    `${margin}</table>`
+  ].join('\n');
+};
+// Returns an HTML table of manager-page activity, sorted by descending failure count,
+// since repeated failures are the signal of suspected abuse most worth surfacing first.
+const getManagerActivityTable = (activity: Record<string, {ok: number; error: number}>): string => {
+  const margin = ' '.repeat(6);
+  const names = Object.keys(activity).sort((a, b) => activity[b]!.error - activity[a]!.error);
+  if (!names.length) {
+    return `${margin}<p>None yet.</p>`;
+  }
+  const rows = names.map(name => {
+    const {ok, error} = activity[name]!;
+    return `${margin}    <tr><td>${name}</td><td>${ok}</td><td>${error}</td></tr>`;
+  });
+  return [
+    `${margin}<table class="allBorder">`,
+    `${margin}  <thead>`,
+    `${margin}    <tr><th>Name</th><th>Successful</th><th>Failed</th></tr>`,
+    `${margin}  </thead>`,
+    `${margin}  <tbody>`,
+    ...rows,
+    `${margin}  </tbody>`,
+    `${margin}</table>`
+  ].join('\n');
+};
+// Returns a page reporting usage metrics, or, if the form has not been submitted (a POST
+// request) yet, a form requesting an authorization code. A GET request, regardless of its
+// query string, never processes a submission, so the action cannot be triggered by GET.
+export const answer = async (_: any, search: string, method: string) => {
+  const searchParams = new URLSearchParams(search);
+  const authCode = searchParams?.get('authCode');
+  let query: Record<string, string>;
+  // If the form has been submitted:
+  if (method === 'POST') {
+    // If the authorization code is invalid:
+    if (!isValidAuthCode(authCode)) {
+      // Report the error, deliberately vague so as not to confirm to an attacker that the
+      // authorization code specifically (as opposed to some other part of the request) is
+      // what was wrong.
+      return {
+        status: 'error',
+        message: 'Invalid request'
+      };
+    }
+    // If clearing the counts was requested:
+    const clearCounts = searchParams.has('clearCounts');
+    const metrics = clearCounts ? await clearMetrics() : await getMetrics();
+    query = {
+      body: [
+        ...(clearCounts ? ['<p>Counts cleared.</p>'] : []),
+        `<p>Counts recorded since ${getDateTimeString(metrics.since)}.</p>`,
+        '<h2>Web page views</h2>',
+        getCategoryTable(metrics.pageViews),
+        '<h2>MCP tool calls</h2>',
+        getCategoryTable(metrics.mcpToolCalls),
+        '<h2>API operation calls</h2>',
+        getCategoryTable(metrics.apiOperations),
+        '<h2>Manager page activity</h2>',
+        '<p>Manager-only pages (linked from <a href="/manage.html">management</a>), excluded from the web page views above. A high failed count for a page may indicate a suspected attack, such as repeated guessing of the authorization code.</p>',
+        getManagerActivityTable(metrics.managerActivity)
+      ].join('\n')
+    };
+  }
+  // Otherwise, i.e. if the form has not been submitted yet:
+  else {
+    query = {
+      body: [
+        '<form action="/metrics.html" method="post">',
+        '  <p><label>',
+        '    Authorization code: <input size="3" minlength="3" maxlength="3" name="authCode" required>',
+        '  </label></p>',
+        '  <p><label><input type="checkbox" name="clearCounts"> Clear counts</label></p>',
+        '  <p><button type="submit">Submit</button></p>',
+        '</form>'
+      ].join('\n')
+    };
+  }
+  // Get the populated template.
+  const answerPage = await populateTemplate(import.meta.dirname, query);
+  // Return the populated page. Submitting a valid authCode also sets a cookie that
+  // excludes this browser's future page views and API calls from usage metrics, on the
+  // premise that a maintainer who has just proven their identity here is very likely the
+  // same person about to manually browse and test the rest of Kilotest.
+  return {
+    status: 'ok',
+    answerPage,
+    ...(method === 'POST' && {
+      setCookie: {
+        name: metricsExclusionCookieName,
+        value: getExclusionCookieValue(),
+        maxAgeSeconds: 30 * 24 * 60 * 60
+      }
+    })
+  };
+};