@jrpool/kilotest 38.1.0 → 39.0.1

package/api/util.js

@@ -0,0 +1,149 @@
+/*
+  util.js
+  Utilities for API requests.
+*/
+
+// IMPORTS
+
+const {sendAlert} = require('../alerts');
+const {
+  getAgoDays,
+  getLatestReportExtracts,
+  getNowStamp,
+  getPlainText,
+  getRandomString,
+  getReportExtract,
+  getReportStats,
+  issuesClassification,
+  objectSort,
+  ruleEngines,
+  updateRecs
+} = require('../util');
+
+// CONSTANTS
+
+const thisHost = exports.thisHost = process.env.THIS_KILOTEST_HOST;
+
+// FUNCTIONS
+
+// Returns uniform metadata for every response.
+exports.getResponseMetadata = () => ({
+  identifier: `${getNowStamp()}-${getRandomString(3)}`,
+  'date and time': new Date().toISOString()
+});
+// Returns facts about the tool collection (Kilotest).
+exports.getToolsFacts = () => ({
+  'name': 'Kilotest',
+  'description': {
+    'what Kilotest does': 'Kilotest tools generate and make available findings about the front-end quality (i.e. accessibility, usability, and standards conformity) of web pages. A Kilotest job generates findings by using Testaro to test a page against more than a thousand rules defined by an ensemble of ten rule engines. Testaro produces a report of the job. The report describes violations of the rules. Kilotest uses Testilo to enhance the report with a classification of the rule violations into about 300 issues. Kilotest makes facts about the issues and the violations retrievable at four levels of granularity.',
+    'how to retrieve findings': {
+      'level 1': 'Use the listReports tool to get a list of available reports.',
+      'level 2': 'Use the listIssues tool to get a list of issues in one report.',
+      'level 3': 'Use the listViolators tool to get a list of elements on one page that were reported in one report as exhibiting one issue.',
+      'level 4': 'Use the listDiagnoses tool to get a list of diagnoses of how one element on one page exhibited one issue in one report.',
+    },
+    'how to generate more findings': {
+      'new testing': 'If no report is available yet about a page, use the requestTest tool to request that it be tested.',
+      'retesting': 'If the listIssues tool shows that the latest report about a page is obsolete, because the page has been revised or for another reason, use the requestRetest tool to request that the page be retested.',
+      'latency': 'Requests for testing and retesting are usually approved and fulfilled within one day.',
+      'confirmation': 'Use the listReports tool to determine whether a requested new report exists. There is currently no process for notification of the outcome of requests.'
+    }
+  },
+  'URL': `${thisHost}/mcp`,
+  'web users can obtain similar functionalities at': thisHost
+});
+// Returns the facts about a rule engine.
+const getRuleEngineFacts = exports.getRuleEngineFacts = ruleEngineID => {
+  const ruleEngineData = ruleEngines[ruleEngineID] || [null, null];
+  return {
+    identifier: ruleEngineID,
+    name: ruleEngineData[0] || null,
+    sponsor: ruleEngineData[1] || null
+  };
+};
+// Returns the facts about rule engines.
+exports.getRuleEnginesFacts = ruleEngineIDSet => {
+  const ruleEnginesFacts = Array.from(ruleEngineIDSet).map(id => getRuleEngineFacts(id));
+  objectSort(ruleEnginesFacts, 'name', 'alpha');
+  return ruleEnginesFacts;
+};
+// Returns the basics about a report, without reading the report.
+exports.getReportBasics = async (timeStamp, jobID) => {
+  // Get the creation time of the report.
+  const reportStats = await getReportStats(timeStamp, jobID);
+  // If the  report does not exist:
+  if (!reportStats) {
+    // Log and return this.
+    console.error(`Report ${timeStamp}-${jobID} does not exist.`);
+    return {
+      error: `Report ${timeStamp}-${jobID} could not be retrieved.`
+    };
+  }
+  // Get an extract of the report.
+  const reportExtract = await getReportExtract(timeStamp, jobID);
+  // If no such report exists:
+  if (reportExtract.error) {
+    // Return this.
+    return reportExtract;
+  }
+  const {url, what} = reportExtract;
+  const latestReportExtracts = await getLatestReportExtracts();
+  // Otherwise, i.e. if the report exists, get whether this report has been superseded.
+  const isSuperseded = latestReportExtracts
+  .every(extract => extract.timeStamp !== timeStamp || extract.jobID !== jobID);
+  const {reportTime} = reportStats;
+  // Get the basics about the report.
+  const basics = {
+    identifier: `${timeStamp}-${jobID}`,
+    'completion date and time': reportTime.toISOString(),
+    'days since the report was completed': getAgoDays(reportTime),
+    'tested web page': {
+      description: what,
+      URL: url
+    },
+    'whether a later report about the same page exists': isSuperseded
+  };
+  // Return them.
+  return basics;
+};
+// Returns the classification of an issue.
+exports.getIssueClassification = issueID => {
+  // Get the issue classification.
+  const issueClassification = issuesClassification[issueID] ?? {};
+  const {summary, wcag, weight, why} = issueClassification;
+  // If the issue is non-ignorable and fully classified:
+  if (
+    issueID
+    && issueID !== 'ignorable'
+    && issueClassification
+    && summary
+    && wcag
+    && [1, 2, 3, 4].includes(weight)
+    && why
+  ) {
+    // Return the classification.
+    return issueClassification;
+  }
+  // Otherwise, i.e. if it is ignorable or not fully classified, return this.
+  return null;
+};
+// Processes a test or retest request.
+exports.processTestRequest = async (testType, what, url, why) => {
+  // Get an email-safe version of the reason.
+  const plainWhy = getPlainText(why);
+  // Update the waiting recommendations as a transaction.
+  const updateResult = await updateRecs(what, url, plainWhy);
+  // If the recommendation was a duplicate:
+  if (updateResult.error === 'duplicate') {
+    // Return this.
+    return {
+      status: 'error',
+      message: 'Duplicate request'
+    };
+  }
+  // Otherwise, i.e. if it was not a duplicate, alert a manager about it.
+  await sendAlert(
+    `Kilotest: new ${testType} recommendation in the API`,
+    `Target: ${what}\nURL: ${url}\nReason: ${plainWhy}`
+  );
+};

package/diagnoses/index.js

@@ -64,7 +64,7 @@ const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, qu
   query.testInfo = testInfo;
   query.issue = issues[issueID]?.summary;
   // If adding the issue summary failed:
-  if (! query.issue) {
+  if (!query.issue) {
     // Populate the query with the reason.
     query.error = 'Issue not found';
     // Stop populating the query.
@@ -77,7 +77,7 @@ const populateQuery = async (issueID, timeStamp, jobID, catalogIndex, pathID, qu
   query.priority = getWeightName(weight);
   query.wcag = `<a href="${getWCAGLink(wcag)}">${wcag}</a>`;
   query.tagName = tagName || 'HTML';
-  if (text && ! ['HTML', 'BODY', 'HEAD', 'SCRIPT', 'STYLE', 'NOSCRIPT'].includes(tagName)) {
+  if (text && !['HTML', 'BODY', 'HEAD', 'SCRIPT', 'STYLE', 'NOSCRIPT'].includes(tagName)) {
     const textString = text.split('\n').join(' … ');
     query.text = `<q>${htmlSafe(textString)}</q>`;
   }

package/docs/{AI-TOOL.md → AI-TOOLS.md}

@@ -1,32 +1,33 @@
-# Kilotest as connector to and creator of AI Tools
+# Kilotest as connector to and collection of AI Tools
 
 ## Introduction
 
 Until 2026 Kilotest was intended, and implemented, as a web application performing a service for human users.
 
-Beginning in May 2026, it [became evident](https://github.com/jrpool/kilotest/issues/2) that Kilotest could also act as a connector to tools for use by language models. Language models are asked for help in all domains, including the domain of software quality. When asked about the front-end quality (accessibility, usability, and standards conformity) of specific web pages, models gave answers without the use of tools. The answers were at best fragmentary but often speculative and fabricated. If Kilotest offered to connect language models to its functionalities as tools, models could give more inexpensive, comprehensive, factual, authoritative, and grounded answers. Every reported defect could be documented and ascribed to one or more specific rule engines in the Kilotest ensemble.
+Beginning in May 2026, it [became evident](https://github.com/jrpool/kilotest/issues/2) that Kilotest could also act as a provider of tools for use by language models. Language models are asked for help in all domains, including the domain of software quality. When asked about the front-end quality (accessibility, usability, and standards conformity) of specific web pages, models gave answers without the use of tools. The answers were at best fragmentary but often speculative and fabricated. If Kilotest provided its functionalities to language models, models could give more inexpensive, comprehensive, factual, authoritative, and grounded answers. Every reported defect could be documented and ascribed to one or more specific rule engines in the Kilotest ensemble.
 
-Given the potential of Kilotest to serve language models and the expected continued growth in the share of questions that are directed to AI platforms that use language models, a decision was made to **make Kilotest discoverable, usable, and, where appropriate, used as a connector to, and creator of, tools for language models**.
+Given the potential of Kilotest to serve language models and the expected continued growth in the share of questions that are directed to AI platforms that use language models, a decision was made to **make Kilotest discoverable, usable, and, where appropriate, used as a provider of tools for language models**.
 
 ## Terms
 
 - **Language model**: A model (e.g., Claude Haiku 4.5, Kimi K2.6, GPT-5.4, Gemini 3.1 Pro) that can consume and generate text, images, or other content.
 - **AI platform**: A platform (e.g., Claude Desktop, Perplexity, ChatGPT, Gemini 3.5 Flash) that gives human users access to the services of language models and connects language models to productivity resources.
-- **tool**: A specialized productivity resource providing a capability that the language model itself needs but does not have. In the OpenAPI specification, the term _operation_ is used to mean tool.
+- **tool**: A specialized productivity resource providing a capability that a language model needs but does not have. In the OpenAPI specification, the term _operation_ is used to mean tool.
 - **connector**: A service that allows AI platforms to enable their language models to discover, evaluate, and use tools.
 
-Tools can be independent of connectors that connect them to language models, but Kilotest can be both. It already provides capabilities to human users; it can re-use the same functions that underlie those capabilities and reconfigure those functions as tools for language models. Kilotest can also provide connectors to those tools for language models and make those connectors compatible with AI platforms.
-
 ## Internal additions
 
-The internal changes that have been made in the Kilotest codebase to support the use of Kilotest as a set of tools and connectors to them are:
+The internal changes that have been made in the Kilotest codebase to support the use of Kilotest as a collection of tools are:
 
 - An API, consisting of:
   - Additions to `index.js`.
   - Additions to `util.js`.
-  - Within directories providing API functionality:
-    - `api.js` modules.
-    - `util.js` modules, if needed, containing resources shared by the `index.js` and `api.js` modules in those directories.
+  - An `api` directory containing:
+    - Eight tool modules.
+    - A `util.js` module with shared functions used by tool modules.
+    - A `routes.js` file.
+    - A `schemas.js` file.
+    - A `testSuite.js` module for testing the API.
 - A [`JSON-LD`](https://json-ld.org/) script in the `index.html` file, providing structured data about the Kilotest API.
 - Additions to the `env.example` file.
 - An `llms.txt` file and an `llms-full.txt` file, documenting the use of Kilotest by language models, conforming to the [llms-txt](https://llmstxt.org/) specification.
@@ -34,12 +35,12 @@ The internal changes that have been made in the Kilotest codebase to support the
 - An `mcp.js` file, providing an MCP server for Kilotest.
 - A `sitemap.xml` file.
 - Additions to the `README.md` file.
-- This `AI-TOOL.md` file.
-- A `researchAgent.js` file, testing the API.
+- This `AI-TOOLS.md` file.
+- A [tutorial on authorizing language models to use Kilotest](https://kilotest.com/qai/).
 
 ## External actions
 
-The external actions that have been taken to support the use of Kilotest as an AI tool are:
+The external actions that have been taken to support the use of Kilotest as a collection of AI tools are:
 
 - A [pull request](https://github.com/public-apis/public-apis/pull/6346/changes) to add Kilotest to the list of public APIs in the `public-apis` repository.
 - A [request](https://rapidapi.com/studio/api_91f2ce07-2572-48bd-a34d-ff01ed6cd039/publish/general) to add Kilotest to the Rapid API Hub.
@@ -53,7 +54,7 @@ The external actions that have been taken to support the use of Kilotest as an A
 
 ## Use cases
 
-The rationale for Kilotest as a connector to tools for language models is set forth in the `llms-full.txt` file and is not repeated here.
+The rationale for Kilotest as a collection of tools for language models is set forth in the `llms-full.txt` file and is not repeated here.
 
 Some common anticipated use cases for this role are:
 
@@ -68,11 +69,7 @@ Among these use cases, case 3 would make it feasible for the user to tell an AI
 
 Use cases 1, 2, 4, 5, and 6 exemplify a widespread expectation and demand for AI platform capability. The commonality is: “I have a question; answer it.” If Kilotest can be employed as an expert for AI platforms in relevant cases, platforms will be more successful in satisfying that demand. At present this is a difficult problem because of platform limitations and a lack of standardization.
 
-## Strategy
-
-The Kilotest project is attempting to solve the just-described problem. That attempt follows the following strategy.
-
-### Increments
+## Implementation strategy
 
 Connector and tool discovery and utilization have been only partly standardized. Major differences in protocols exist among model and platform families. Therefore, small testable increments of improvement in the connector and tool functionality of Kilotest can best be defined by model and platform family and by use-case characteristics.
 
@@ -80,7 +77,11 @@ One benefit of such incrementalism is that, after the first increment succeeds,
 
 Another benefit is that subsequent increments can be defined incrementally rather than in advance. Lessons learned from the work on each increment can inform the choice of what to work on next.
 
-#### Increment 1
+## Implementation history
+
+The Kilotest project has been attempting to solve the just-described problem. A chronology of this work follows.
+
+### Increment 1
 
 In the first increment, the objective was to make Anthropic Claude models use Kilotest to help them answer questions from developers using the Claude Desktop application for use case 3, where the code in question is already deployed as a public web page.
 
@@ -88,7 +89,7 @@ A locally installed MCP server, `@ivotoby/openapi-mcp-server`, was used. It turn
 
 The model behavior motivated an architectural change in the MCP server and improvements in the naming and description of the Kilotest tools before the next increment.
 
-#### Increment 2
+### Increment 2
 
 Increment 2 repeated the exercise of increment 1 with the `claude.ai` web application instead of Claude Desktop, and with the improvements indicated by increment 1 made.
 
@@ -105,7 +106,7 @@ The trial in increment 2 suggested some opportunities for further improvements:
 - Clearer instructions about the sequencing of tool calls
 - Addition of instructions to attribute the interpretive additions by the model to the model or its other sources rather than to Kilotest.
 
-#### Increment 3
+### Increment 3
 
 After the above-described improvements were made, a similar exercise, with the home page of a company named “CMT Services”, was performed again in Claude Desktop. In this case Claude Haiku 4.5 succeeded in recognizing the relevance of Kilotest and used its tools, though imperfectly. Instead of using Kilotest to check for an existing report on the page, the model searched for it in a cached file with:
 
@@ -137,7 +138,7 @@ The misbehavior of Claude Haiku suggested that further improvements in the instr
 - a warning not to use cached lists of available reports
 - stronger advice to avoid implicitly attributing interpretations, judgments, and advice to Kilotest.
 
-#### Increment 4
+### Increment 4
 
 The first trial in which the user asked about a page **without** an available report repeated the conditions of Increment 3, except for the web page in question and the model. The model here was Claude Haiku in Extended mode, an option available with `claude.ai` but not Claude Desktop.
 
@@ -160,7 +161,7 @@ The most notable fact was that Claude Haiku converted a long workflow to a short
 
 This result suggested that the instructions should be revised to describe, as a valid option, a short workflow ending with a recommendation to the user to get the test results by self-service after the model submits a testing recommendation.
 
-#### Increment 5
+### Increment 5
 
 In the next increment the AI platform was changed to [Perplexity Pro](https://www.perplexity.ai/) on the web. With the platform UI, two configuration steps were taken:
 
@@ -181,8 +182,35 @@ For both models, the same exercise was also attempted with the second configurat
 
 The [Perplexity documentation](https://www.perplexity.ai/help-center/en/articles/13915507-adding-custom-remote-connectors) states that as of now the installed Perplexity application for macOS does not support remote connectors.
 
-#### Increment 6
+### Increment 6
 
 Once the compatibility of Kilotest as a connector with AI platforms of two different vendors had been confirmed, the focus of work returned to completion of the missing endpoints of the API. The first was an endpoint that would let a model efficiently search for an available report about a particular web page.
 
-The new `summarizeQualityOfMatchingWebPages` tool provides inputs for fragments of a description and of a URL hostname of a web page and responds with summary data about all and only the available reports of pages that match at least one of those fragments, where matching means either including or being included by the provided fragment, case-insensitively.
+The new `summarizeQualityOfMatchingWebPages` tool provided inputs for fragments of a description and of a URL hostname of a web page and responds with summary data about all and only the available reports of pages that match at least one of those fragments, where matching means either including or being included by the provided fragment, case-insensitively.
+
+### Increment 7
+
+Further experimentation made it clear that Increment 6 had been a detour. The problem with Increment 6 was that it does not improve efficiency by eliminating the need to get a list of all reports. Models must get a list of all reports in order to be confident that the answer to the question “Is a report available yet?” is correct. Any algorithm that matches a page description and a URL with those of existing reports is error-prone, organization names change, website designs change, URLs change, and organizations are related to one another. Typically, a desired report does not yet exist. In that case, the model needs the full list of reports in order to understand the naming convention for pages and to adhere to that convention when submitting a request to have the page of interest tested.
+
+For this reason, the tool being developed in increment 6 was deleted.
+
+### Increment 8
+
+The next increment reorganized the API by placing API-related code into a new `api` directory.
+
+### Increment 9
+
+The collection of tools was enlarged to a total of eight tools:
+
+- listReports
+- listIssues
+- listViolators
+- listDiagnoses
+- getReport
+- requestRetest
+- requestTest
+- requestFeature
+
+## Increment 10
+
+In the next increment, API code was refactored and the tools were made more consistent with one another.

package/docs/SERVICE.md

@@ -203,8 +203,9 @@ kilotest.com {
   }
   @allowedPOST {
     method POST
-    path /mcp /retestRec.html/* /testRec.html/* /recAction.html /reannotate.html /wcagRenew.html /api/* /tutorialComment.html
+    path /mcp /retestRec.html/* /testRec.html /recAction.html /reannotate.html /wcagRenew.html /api/* /tutorialComment.html
   }
+  # Respond to OPTIONS requests.
   @allowedOPTIONS method OPTIONS
   handle @allowedOPTIONS {
     header {
@@ -214,6 +215,8 @@ kilotest.com {
     }
     respond 204
   }
+  # Redirect any /qai request to /qai/.
+  redir /qai /qai/ 301
   # Truncate any initial /qai from the forwarded path.
   handle_path /qai* {
     # Forward any QAI request to port 3001.

package/docs/decisions/001-api-design.md

@@ -1,49 +1,51 @@
 ---
-date: 2026-07-09
-status: proposed
+date: 2026-07-14
+status: accepted
 ---
 
 # API design
 
 ## Context and problem
 
-The UI of Kilotest offers human non-manager users opportunities to influence and to interrogate the data managed by the application. The data are organized into reports. Each report contains the results of one session in which 10 rule engines were used for testing the front-end quality of one web page.
+Kilotest manages data that are organized into reports. Each report contains the results of one session in which multiple rule engines were used for testing the front-end quality of one web page.
+
+The UI offers users opportunities to influence and interrogate the data managed by the application. Users do this with input via links and forms. The output is HTML. UI users are typically humans.
 
 The opportunities to interrogate the data in the UI are organized as a hierarchy of granularity:
 
 1. All available reports
-2. All issues from one report
-3. All violators of one issue from one report
-4. All diagnoses for one violator of one issue from one report
+2. Details of one report, including summaries of the reported issues.
+3. Details of one issue, including summaries of its violators.
+4. Details of one violator, including its diagnoses.
 
-A violator is a particular element (HTML, SVG, etc.) on the page that some rule engine reported having violated a particular rule of the rule engine. If the rule classifier used by Kilotest classifies that rule as belonging to a particular issue, then that element is referred to as a violator of that issue.
+A violator of an issue is a particular element (HTML, SVG, etc.) on the page. When a rule engine reports a violation of one of its rules, the rule engine usually identifies an element as responsible for that violation. If the rule classifier used by Kilotest classifies that rule as belonging to a particular issue, then Kilotest refers to that element as a violator of that issue.
 
-This hierarchical design is based on a realistic assumption about how human users interact with Kilotest. They visit its home page and begin what to them is a session. Although requests to Kilotest are stateless, the response at each level includes links that contain all the specifications required for a request at the next level. For example, the response to a request at level 3 includes, for each violator, a link that makes a request at level 4 and includes identifiers of the page, the issue and the violator.
+This hierarchical design is based on a realistic assumption about how human users interact with Kilotest. They visit its home page and begin what to them is a session. Although requests to Kilotest are stateless, the response at each level includes links that contain all the specifications required for a request at the next level. For example, the response to a request at level 3 includes, for each violator, a link that makes a request at level 4 and includes identifiers of the page, the issue, and the violator.
 
-The responses to requests in the UI at any level include a summary of some of the information about the previous levels. For example, a response at level 3 includes a summary of information about the report and about the issues in that report. The prior-level summarized information is based on assumptions about what a typical human user would value. For example, at level 3, the prior-level information does not summarize all the available reports, but does include a list of the rule engines that succeeded and failed in testing the page and the counts of issues reported.
+The responses to requests in the UI at any level include some of the information about the previous levels. For example, a response at level 3 includes a summary of information about the report and about the issues in that report. The prior-level summarized information is based on assumptions about what a typical human user would value. For example, at level 3, the prior-level information does not summarize all the available reports, but does include a list of the rule engines that succeeded and failed in testing the page and the counts of issues reported.
 
-These assumptions for human UI users may or may not be valid for how LLMs interact with the Kilotest API. Those interactions may vary depending on the tasks that they are performing. Consider three scenarios:
+These assumptions for human UI users may or may not be valid for how API consumers prefer to interact with a Kilotest API. Those interactions may vary depending on the tasks that they are performing. Consider three scenarios:
 
-1. A user asks an LLM for all available information about the front-end quality of a particular web page.
-2. A user asks an LLM for all the available information about issues related to a particular topic, such as keyboard navigation, for a particular web page.
-3. A user asks an LLM to summarize the front-end quality of a particular web page; having consumed the summary, asks the LLM for a summary of information about one of the reported issues; having consumed that summary, asks the LLM for a summary of information about one of the reported violators of that issue; and, having consumed that summary, asks the LLM for a summary of information about the diagnoses for that violator of that issue.
+1. A user asks a language model for all available information about the front-end quality of a particular web page.
+2. A user asks a language model for all the available information about issues related to a particular topic, such as keyboard navigation, for a particular web page.
+3. A user asks a language model to describe the front-end quality of a particular web page. Having consumed the description, the user asks the model to describe one of the reported issues. Having consumed that description, the user asks the model to describe one of the reported violators of that issue. Having consumed that description, the user asks the model to describe the diagnoses of that violation.
 
-Scenario 1 seems to justify an API that can aggregate all the information about the latest report about the page into a single response.
+Scenario 1 seems to justify an API that can aggregate all information from all available reports about the page into a single response.
 
-Scenario 2 seems to justify an API that offers enough information about all the available reports and all the defined issues to allow the LLM to make a request at level 3 for a particular issue in a particular report.
+Scenario 2 seems to justify an API that filters out from the scenario-1 response all information about issues unrelated to the specified topic.
 
-Scenario 3 seems to justify an API that mimics the UI in structure, but perhaps not in the prior-levels information provided at each level.
+Scenario 3 seems to justify an API that mimics the UI in structure.
 
-Even these apparently justified API designs depend on the assumption that users mean what they say, particularly in scenario 1. A user who asks for **all** available information about the front-end quality of a particular web page may do so because of ignorance about how voluminous that information is, and the real wish may be more conditional: all the information if there is not much, but otherwise a summary so I can select what I want to learn more about.
+Even these apparently justified API designs depend on the assumption that users mean what they say. For example, in scenario 1 a user asks for **all** available information about the front-end quality of a particular web page. The user may really want all that information only if it is not too voluminous, and otherwise want a summary.
 
-Uncertainty about how LLMs will interact with the API creates a design problem. How should the API be further developed?
+Uncertainty about how consumers will interact with the API creates a design problem: How should the API be further developed?
 
-As of the date of this decision, a partial initial API version already exists: an API that mimics the UI for levels 1, 2, and 3. For each level, the MCP server provides a tool.
+As of the date of this decision, a partial initial API version already exists: an API that mimics the UI for levels 1, 2, and 3. It is exposed as a traditional HTTP REST API and also as a connector to a collection of MCP tools. For each level, the API provides an operation.
 
 ## Considered options
 
-1. **Iterate and instrument.** (1) Finish redesigning the tools at levels 1, 2, and 3, to represent the hypothesized typical needs of LLM consumers; then (2) add a tool that implements level 4; then (3) add a tool that asks LLMs to comment on how the API design could better satisfy their needs; then (4) add metrics to make API usage and submitted LLM comments available to Kilotest managers.
-2. **Research first.** (1) Pause further API development while doing fundamental research to predict LLM needs; then (2) resume API and metrics development, including API redesign if the research results warrant.
+1. **Iterate and instrument.** (1) Finish redesigning the operations at levels 1, 2, and 3, to represent the hypothesized typical needs of language-model consumers. Then (2) add an operation that implements level 4. Then (3) add an operation that collects comments from consumers on how the API design could better satisfy their needs. Then (4) add metrics to make API usage and submitted comments available to Kilotest managers.
+2. **Research first.** (1) Pause further API development while doing fundamental research to predict consumer needs. Then (2) resume API and metrics development, including API redesign if the research results warrant.
 
 ## Decision
 
@@ -51,60 +53,68 @@ The decision is to adopt **option 1: iterate and instrument**.
 
 ## Empirical basis
 
-The decision is based on the fact that, even without a pause for fundamental research, there is enough empirical evidence to justify this decision.
+The decision is based on the working hypothesis that the typical consumer will be a language model (most often a foundation model) performing a task for a human user. Even without a pause for fundamental research, there is enough empirical evidence to justify this decision.
 
 ### Completion of the 4-level hierarchy
 
-There is empirical support for an API that assumes scenario 3, i.e. incremental rather than one-shot interrogation. Both single-shot comprehensive requests and incremental drill-downs occur in significant numbers in real-world LLM use.
+There is empirical support for an API optimized for scenario 3, i.e. incremental rather than one-shot interrogation. Both single-shot comprehensive requests and incremental drill-downs occur in significant numbers in real-world language-model use.
 
-- Jian et al. (2024) studied experienced ChatGPT users and found that **43.8% of prompts were single-component**, **38.2% were dual-component**, and only **18.0% were multi-component**. A request such as scenario 1 ("tell me everything about page X") or scenario 2 ("tell me about defect class X on page Y") is structurally common because it is typically a single-component prompt.
-- At the same time, follow-up refinement is routine. Mysore et al. (2025) analyzed large-scale Bing Copilot and WildChat writing sessions and report: “Rather than passively accepting output, users actively refine, explore, and co-construct text.” This matches scenario 3.
+- Jian _et al._ (2024) studied experienced ChatGPT users and found that **43.8% of prompts were single-component**, **38.2% were dual-component**, and only **18.0% were multi-component**. A request such as scenario 1 (“tell me everything about page X”) or scenario 2 (“tell me about defect class X on page Y”) is structurally common because it is typically a single-component prompt.
+- However, follow-up refinement is frequent. Mysore _et al._ (2025) analyzed large-scale Bing Copilot and WildChat writing sessions and report: “Rather than passively accepting output, users actively refine, explore, and co-construct text.” This matches scenario 3.
 
-For scenario 3, current LLM tool-use patterns do not require the model to restart at level 1. Anthropic's tool-use documentation describes an agentic loop in which the full conversation history is kept and “every turn sees the complete prior context”. Cohere’s multi-step tool-use documentation explains that the model can use information learned from one tool call to parameterize a subsequent call. MCP’s typed tool schemas support this pattern. Therefore the existing hierarchy can be expected to be practical in scenario 3.
+For scenario 3, current patterns do not require the consumer to restart at level 1. Anthropic’s tool-using agent documentation describes an agentic loop in which the full conversation history is kept and “every turn sees the complete prior context”. Cohere’s multi-step tool-use documentation explains that the model can use information learned from one tool call to parameterize a subsequent call. MCP’s typed tool schemas support this pattern. Therefore the existing hierarchy can be expected to be practical in scenario 3.
 
 ### Usage and suggestion metrics
 
-There is also empirical support for modesty in any estimate of the adequacy of the current or any other single API design. As found by Zhu et al. (2025), “users do not transfer existing conversational norms to LLM interactions, but instead develop new registers with distinct pragmatic features.” Metrics will help to inform the contributors to the API about how LLMs are actually using it and how they want it to be improved.
+There is also empirical support for modesty in any estimate of the adequacy of the current or any other single API design. As found by Zhu _et al._ (2025), “users do not transfer existing conversational norms to LLM interactions, but instead develop new registers with distinct pragmatic features.” Metrics will help to inform the contributors to the API about how consumers are actually using it and about proposed improvements.
 
 ## Cost and context-window constraints
 
-A user-friendly rendering of an entire Kilotest report is estimated at about 5 MB of text, or roughly 1.25 million tokens. This exceeds the 200,000-token context window of Claude 3.5 Sonnet, so the model cannot consume the whole report in one prompt. If it could be passed whole, each pass would cost roughly $3.75 in input tokens at current pricing, and every subsequent question about it would require re-sending it. By contrast, a four-level drill-down through report → issue → violator → diagnosis costs roughly $0.07 for a typical path. An intuitive guess is that a typical conversation would involve 1 report, 2 issues, and 3 violators per issue. Delivering such results with the full-report method would require 2 issue passes and 6 violator passes beyond the initial pass, for a total of about $33.75. Doing so incrementally would cost about $0.26, which is only **1/130th** of the cost of the same result with the full-report strategy. Even if a user initially wanted a comprehensive report and then wanted an LLM to find and interpret it, the impact of that strategy on cost would likely motivate the user to switch to an incremental strategy.
+A user-friendly rendering of an entire Kilotest report is estimated at about 5 MB of text, or roughly 1.25 million tokens. For a cost estimate, we can use one major frontier model as of the date of this decision (Claude 3.5 Sonnet). On that basis, the report size exceeds the 200,000-token context window of the model, so the model cannot consume the whole report in one prompt. If it is passed chunked, each pass will cost roughly $3.75 in input tokens. Every subsequent question about it will require re-sending it. By contrast, a four-level drill-down through report → issue → violator → diagnosis costs roughly $0.07 for a typical path.
+
+These estimates can be applied to a hypothetically typical conversation. It would involve one report, two issues, and six violators (two per issue). The user would not ask about diagnoses. Delivering such results with the full-report method requires one pass to identify the report, two passes to describe the issues, and six passes to describe the violators, for a total of about $33.75. Doing so incrementally costs about $0.26, which is only **1/130th** of the cost of the same result with the full-report strategy.
+
+On the basis of these cost estimates, even if a user initially wanted a comprehensive report and then wanted a language model to interpret it, the impact of that strategy on cost would likely motivate the user to switch to an incremental strategy as a default.
 
 ## Sub-decision: ancestor information in level responses
 
-Given this decision, a secondary decision to make is what information from prior levels to repeat in a response. Three alternatives were considered:
+Given this decision, a secondary decision to make is what information from prior levels to repeat in a response. Four alternatives were considered:
+
+1. **Identifiers only:** The report, issue, and violator would be described by their unique identifiers only. The consumer would use the identifiers to retrieve from its conversation-history context any forgotten information needed about them.
+2. **Identifiers plus labels:** Ancestor entities (report, issue, and/or violator) would be described only by their unique identifiers and human-interpretable labels. If the consumer needs other forgotten facts about them, it would use the identifiers to retrieve those facts by repeating ancestor requests.
+3. **Currently pertinent facts:** The facts from ancestor requests would be repeated if, and only if, the consumer could need them for interpreting the response to the current-level request. For example, if the current-level request is about one report, facts about other reports are not pertinent, but any facts about the report are pertinent. Likewise for one issue or one violator.
+4. **Cumulative facts:** All the information from the prior responses would be repeated, so that any needed information would be available in the current response and no search of the conversation history would be needed.
 
-1. **Identifiers only:** The report, issue, and violator would be described by their unique identifiers only. The LLM would use the identifiers to retrieve from its conversation-history context any information needed about them.
-2. **Identifiers plus labels:** The report, issue, and violator would be described by their unique identifiers and human-interpretable labels. If the LLM needs more information about them than what their labels provide, the LLM would use the identifiers to retrieve from its conversation-history context that additional information.
-3. **Cumulative responses:** All the information from the prior responses would be repeated, so that any needed information would be available in the current response and no search of the conversation history would be needed.
+The sub-decision was **currently pertinent facts** (option 3). This decision was based on cost and reliability, as follows.
 
-The sub-decision was **identifiers plus labels** (option 2). This decision was based on cost and reliability, as follows.
+- **Cost:** Option 3 is slightly more costly than option 2 when all pertinent facts are remembered, because it provides duplicate information. But option 3 is substantially less costly than option 2 if needed facts are forgotten, because option 3 avoids the need to repeat an ancestor request to re-retrieve forgotten facts. Option 4 is much more costly than option 3, because of the much larger response bodies and the inclusion of much irrelevant information in responses.
+- **Quality:** Option 2 provides better locality than option 1, and option 3 provides better locality than option 2. Locality improves the probability that a language model will actually make use of pertinent facts instead of ignoring them or fabricating hallucinated replacements for them.
+- **Reliability:** Option 1 entails a greater risk of information loss and hallucination than option 2. With option 1, the consumer could be unable to retrieve the needed information using identifiers if the conversation history were truncated, if an operation were invoked in isolation, or if the model were asked later to summarize findings in natural language.
 
-- **Cost:** Option 3 would probably make a four-level drill-down several times as expensive as option 2, because of the large response volumes.
-- **Reliability:** Option 1 would entail a greater risk of information loss and hallucination than option 2, because the LLM could be unable to retrieve the needed information using identifiers if its history were truncated, if a tool were invoked in isolation, or if the model were asked later to summarize findings in natural language.
+No sub-decision option is optimal under all possible conditions, but the working hypothesis about prevailing use patterns favors option 3.
 
 ## Implementation plan
 
-- The existing MCP tools for levels 1, 2, and 3 will be revised to remove excess information about prior levels and include only identifiers and labels for them.
-- Level 4 will be implemented as an MCP tool that returns the diagnoses for one violator of one issue in one report.
-- A suggestions tool will be provided so that LLM-based clients can report friction or suggest improvements to the API design.
-- Usage metrics will be defined, captured, and recorded so that Kilotest managers can observe which levels, issues, and patterns are most common and what suggestions have been received.
+- The existing request handlers for levels 1, 2, and 3 will be revised. After revision, they will omit excess information about prior levels and include all currently pertinent facts.
+- Level 4 will be implemented. It will return the diagnoses for one violator of one issue in one report.
+- A comment operation will be provided. This will allow consumers to report friction or suggest improvements to the API design.
+- Usage metrics will be defined, captured, and recorded. This will allow Kilotest managers to observe which levels, issues, and patterns are most common and what comments have been received.
 - If the empirical data later show that the API does not serve some common scenario well, the API will be redesigned accordingly.
 
 ### Risks and mitigations
 
-- **Risk:** The hierarchy may not match the way LLMs actually want to query the data.
-  - **Mitigation:** The suggestions tool and usage metrics will surface mismatches early.
-- **Risk:** The cost estimate assumes a well-behaved client that preserves conversation history; a poorly implemented client could still incur higher-than-necessary costs.
-  - **Mitigation:** MCP clients are expected to maintain message history by protocol; the ancestor labels provide a minimal human-friendly substitute.
-- **Risk:** Scenario 1 users may still want an aggregated "everything about page X" response.
-  - **Mitigation:** A future tool can be added to aggregate the hierarchy on the server side and return a structured summary, but that decision is deferred until usage and suggestions data justify it.
+- **Risk:** The hierarchy may not match the way consumers actually want to query the data.
+  - **Mitigation:** The comment operation and usage metrics will surface mismatches early, allowing Kilotest managers to make data-driven decisions about API improvements.
+- **Risk:** The cost estimates assume the typical consumer preserves conversation history. A poorly implemented consumer could still incur higher-than-necessary costs.
+  - **Mitigation:** It is hypothesized that typical consumers will be models that maintain message history. When they do not, the ancestor labels provide a minimal human-friendly substitute that facilitates history reconstruction when needed.
+- **Risk:** Scenario 1 consumers may still want an aggregated “everything about page X” response.
+  - **Mitigation:** A future operation can be added to aggregate the hierarchy on the server side and return a structured summary. That decision is deferred until usage and comment data justify it.
 
 ## References
 
-- Jian, J., et al. (2024). [Understanding User Prompting Behavior in Generative AI](https://dl.acm.org/doi/abs/10.1002/pra2.1318).
-- Mysore, S., et al. (2025). [Prototypical Human-AI Collaboration Behaviors from LLM-Assisted Writing in the Wild](https://aclanthology.org/2025.emnlp-main.852/).
-- Zhu, S., et al. (2025). [Show or Tell? Modeling the evolution of request-making in human-LLM conversations](https://arxiv.org/html/2508.01213v2).
+- Jian, J., _et al._ (2024). [Understanding User Prompting Behavior in Generative AI](https://dl.acm.org/doi/abs/10.1002/pra2.1318).
+- Mysore, S., _et al._ (2025). [Prototypical Human-AI Collaboration Behaviors from LLM-Assisted Writing in the Wild](https://aclanthology.org/2025.emnlp-main.852/).
+- Zhu, S., _et al._ (2025). [Show or Tell? Modeling the evolution of request-making in human-LLM conversations](https://arxiv.org/html/2508.01213v2).
 - Anthropic. [Tutorial: Build a tool-using agent](https://platform.claude.com/docs/en/agents-and-tools/tool-use/build-a-tool-using-agent).
-- Cohere. [Multi-step Tool Use](https://docs.cohere.com/v1/docs/multi-step-tool-use).
+- Cohere. [Multi-step Tool Use](https://docs.cohere.com/docs/tool-use-usage-patterns#multi-step-tool-use).
 - [Model Context Protocol specification](https://modelcontextprotocol.io/specification/2025-06-18/server/tools).

package/eslint.config.mjs

@@ -8,19 +8,29 @@ import {defineConfig} from "eslint/config";
 export default defineConfig([
   {
     ignores: [
-      "IDEAS.md",
+      "docs/IDEAS.md",
       "package-lock.json"
     ]
   },
   {
     files: ["**/*.{js,mjs,cjs}"],
-    plugins: { js },
+    plugins: {
+      js
+    },
     extends: ["js/recommended"],
     languageOptions: {
       globals: globals.node
     },
     rules: {
-      "no-control-regex": "off"
+      "no-control-regex": "off",
+      "rest-spread-spacing": ["error", "never"],
+      "space-unary-ops": [
+        "error",
+        {
+          "words": true,
+          "nonwords": false
+        }
+      ]
     }
   },
   {

package/index.html

@@ -58,7 +58,7 @@
     <main>
       <h1>Kilotest</h1>
       <p>Kilotest performs more than a thousand tests on a web page.</p>
-      <p><a href="/tutorial.html">New to accessibility testing? Start with the tutorial.</a></p>
+      <p>New to accessibility testing? <a href="/tutorial.html">Start with the tutorial.</a></p>
       <details>
         <summary>More than a thousand? Really?</summary>
         <h2>Ensemble testing</h2>