@arjunkhera/atlas 0.3.12 → 0.3.14

package/tests/drivers/browser.mjs

@@ -0,0 +1,364 @@
+// The browser driver. It opens a web page in Chromium and reads it as a person
+// sees it. It is built on Playwright, which is an optional peer: the kit loads
+// and every other driver works when Playwright is not installed. Only a scenario
+// that uses a `browser` way in needs it. Without it the result is `blocked`, and
+// the reason names the cause.
+//
+// The seven duties of a driver (design section 7.2):
+//   1. It refuses an address that guards.hosts does not allow (`blocked`). It checks
+//      the address it opens, the address the page ends on after a redirect, and each
+//      request the page makes: a request to another host is stopped and recorded.
+//   2. It sends a credential only to the origin it belongs to. A cookie is set for
+//      the origin of the way in. A header goes only with requests to that origin.
+//   3. It records each call and each answer. It also records the console errors,
+//      the page errors and the failed requests of the page, as evidence.
+//   4. It removes run secrets and named secrets from what it records, by value. A
+//      screenshot is a picture, so the driver masks each part of the page that
+//      shows a secret before it takes the picture.
+//   5. It returns the raw answer: { status, body, isError, headers, raw }.
+//   6. It signs in as a persona, never as the person who runs the test. Each persona
+//      has its own browser context, so no cookie moves from one persona to another.
+//   7. It closes what it opened: every context and the browser.
+//
+// A request has one of these forms:
+//   { open: '/week?x=1' }                     open a path of the base address (or a full address)
+//   { read: finder, timeout?, optional? }     read the parts that match the finder
+//   { click: finder }                         click the first part that matches
+//   { fill: finder, value }                   type into the first part that matches
+//   { screenshot: 'view' }                    keep a picture of the page as evidence
+//   { method, url, headers?, body? }          a plain HTTP call in the context of the persona,
+//                                             for a dev sign-in. The cookie it gets stays in that context.
+//
+// A finder names a page part the way a person would, from the map:
+//   { role, name?, exact? }  { text, exact? }  { label }  { css }  { hasText, has: finder, within: finder, nth }
+// `css` is the last resort, for a part with no role and no text of its own.
+//
+// `read` waits up to `timeout` (10 seconds) for the first part to show. When none
+// shows, it does not throw: it answers { count: 0, texts: [], found: false }, so an
+// assertion fails and the run reads `fail`, not `blocked`.
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { Blocked } from '../errors.mjs';
+import { hostAllowed, requireAllowed, sameOrigin } from '../guards.mjs';
+import { clip } from '../evidence.mjs';
+
+const DEFAULT_TIMEOUT = 10000;
+const NAVIGATION_TIMEOUT = 30000;
+const INTERNAL = /^(?:about:|data:|blob:|chrome-error:)/;
+
+// The reason to give when the import of the optional peer fails.
+export function explainMissing(error) {
+  const missing = error?.code === 'ERR_MODULE_NOT_FOUND' || error?.code === 'MODULE_NOT_FOUND';
+  if (missing && /playwright/.test(String(error.message))) {
+    return new Blocked('the browser driver needs the package "playwright" and it is not installed here. Install it in the repo (npm install --save-dev playwright), then install the browser (npx playwright install chromium)');
+  }
+  return new Blocked(`the package "playwright" could not be loaded: ${error?.message ?? error}`);
+}
+
+// Loads the optional peer. The import runs here, in the kit copy of the repo, so it finds the
+// "playwright" of that repo.
+export async function loadPlaywright() {
+  try {
+    return await import('playwright');
+  } catch (error) {
+    throw explainMissing(error);
+  }
+}
+
+// A finder can hold a regular expression. The record shows it as text, because JSON drops it.
+const show = (finder) => JSON.parse(JSON.stringify(finder, (key, value) => (value instanceof RegExp ? value.toString() : value)));
+
+const normal = (text) => String(text ?? '').replace(/\s+/g, ' ').trim();
+
+// Builds a Playwright locator from a finder.
+export function locate(scope, finder) {
+  if (!finder || typeof finder !== 'object') throw new Error('a finder must be an object such as { role: "listitem" }');
+  let one = finder.within ? locate(scope, finder.within) : scope;
+  const exact = finder.exact === true;
+  if (finder.role) one = one.getByRole(finder.role, finder.name === undefined ? {} : { name: finder.name, exact });
+  else if (finder.text !== undefined) one = one.getByText(finder.text, { exact });
+  else if (finder.label !== undefined) one = one.getByLabel(finder.label, { exact });
+  else if (finder.css) one = one.locator(finder.css);
+  else throw new Error('a finder needs a role, a text, a label or a css');
+  const filter = {};
+  if (finder.hasText !== undefined) filter.hasText = finder.hasText;
+  if (finder.has) filter.has = locate(scope.page ? scope.page() : scope, finder.has);
+  if (Object.keys(filter).length) one = one.filter(filter);
+  if (finder.nth !== undefined) one = one.nth(Number(finder.nth));
+  return one;
+}
+
+// `playwright` is for a test: { chromium } with the same calls. `load` replaces the import, for a test of the missing peer. `executablePath` can come from
+// the environment (ATLAS_BROWSER_EXECUTABLE) when the machine keeps its browser in an odd place.
+export async function createBrowserDriver({ way, base, hosts, record: rawRecord, redactor, attach, playwright, load = loadPlaywright, timeoutMs = DEFAULT_TIMEOUT }) {
+  if (!base) throw new Error(`the way in "${way}" has no base address`);
+  const origin = new URL(base).origin;
+  requireAllowed(base, hosts, `the way in "${way}"`);
+  const record = rawRecord && ((call) => rawRecord(redactor ? redactor.deep(call) : call));
+  const lib = playwright ?? await load();
+  const engine = lib.chromium ?? lib.default?.chromium;
+  if (!engine) throw new Blocked('the package "playwright" has no chromium engine');
+
+  let browser = null;
+  let closed = false;
+  const contexts = new Map(); // persona name -> { context, page, persona }
+
+  async function launch() {
+    if (browser) return browser;
+    const executablePath = process.env.ATLAS_BROWSER_EXECUTABLE || undefined;
+    try {
+      browser = await engine.launch({ headless: true, executablePath });
+    } catch (error) {
+      throw new Blocked(`the browser could not start: ${String(error?.message ?? error).split('\n')[0]}. Install it with "npx playwright install chromium", or set ATLAS_BROWSER_EXECUTABLE`);
+    }
+    return browser;
+  }
+
+  // Addresses that the guards stopped. A stopped request counts once, as stopped: it is no failed request.
+  const stopped = new Set();
+  const event = (persona, kind, data) => record?.({ way, persona, request: { event: kind }, response: data });
+
+  // The credential of a persona: cookies go into the context for the base origin; other headers (a bearer
+  // token) go only with a request to the base origin. Returns { cookies, extra }.
+  function credentialOf({ headers = {}, credential = null }) {
+    const extra = {};
+    const cookies = [];
+    for (const [name, value] of Object.entries(headers)) {
+      if (name.toLowerCase() === 'cookie') {
+        for (const part of String(value).split(';').map((x) => x.trim()).filter(Boolean)) {
+          const at = part.indexOf('=');
+          cookies.push({ name: part.slice(0, at), value: part.slice(at + 1), url: origin });
+        }
+      } else extra[name.toLowerCase()] = String(value);
+    }
+    if (credential) extra.authorization = `Bearer ${credential}`;
+    return { cookies, extra };
+  }
+
+  async function contextOf(persona, given = {}) {
+    const key = persona ?? '-';
+    const { cookies, extra } = credentialOf(given);
+    if (contexts.has(key)) {
+      // A credential that comes after the context was made must not be dropped without a word.
+      const entry = contexts.get(key);
+      for (const [name, value] of Object.entries(extra)) {
+        if (entry.extra[name] !== value) throw new Blocked(`the browser context of "${persona ?? 'no persona'}" was made before its credential (${name}) came, so the credential would be dropped. Give the credential on the first call of the persona`);
+      }
+      if (cookies.length) await entry.context.addCookies(cookies);
+      return entry;
+    }
+    // Service workers could answer a request with no route, so they are blocked.
+    const context = await (await launch()).newContext({ viewport: { width: 1280, height: 900 }, serviceWorkers: 'block' });
+    const entry = { context, page: null, persona, key, extra };
+    if (cookies.length) await context.addCookies(cookies);
+    const stop = (url, why, kind = 'request-stopped') => { if (kind === 'request-stopped') stopped.add(url); event(persona, kind, { url: redactUrl(url), why }); };
+    const refusal = `guards.hosts allows only ${hosts.join(', ')}`;
+    const allowedUrl = (url) => { try { return hostAllowed(new URL(url).hostname, hosts); } catch { return false; } };
+
+    // Each request goes through here, and so does each hop of a redirect. The route sees the first URL
+    // of a chain only, so the driver does not let the browser follow a redirect: it fetches the request
+    // with no redirect, checks the Location, and hands the browser the answer. The browser then follows
+    // the hop as a new request, and that request comes back here to be checked again. The credential goes
+    // only with a request whose own origin is the base origin, so it never follows a hop to another host.
+    await context.route('**/*', async (route) => {
+      const request = route.request();
+      const url = request.url();
+      if (INTERNAL.test(url)) return route.continue();
+      if (!allowedUrl(url)) { stop(url, refusal); return route.abort('blockedbyclient'); }
+      const headers = { ...request.headers() };
+      if (sameOrigin(url, origin)) Object.assign(headers, entry.extra);
+      else for (const name of Object.keys(entry.extra)) delete headers[name];
+      let response;
+      try {
+        response = await route.fetch({ headers, maxRedirects: 0 });
+      } catch (error) {
+        stop(url, `the request failed: ${String(error?.message ?? error).split('\n')[0]}`, 'request-failed');
+        return route.abort('failed');
+      }
+      const status = response.status();
+      const location = status >= 300 && status < 400 ? response.headers()['location'] : null;
+      if (location) {
+        let next = null;
+        try { next = new URL(location, url).href; } catch { /* a bad Location is refused below */ }
+        if (!next || !allowedUrl(next)) { stopped.add(url); stop(next ?? location, `a redirect from ${new URL(url).pathname} leads to an address that ${refusal}`); return route.abort('blockedbyclient'); }
+      }
+      return route.fulfill({ response });
+    });
+    // A WebSocket is not a request of the route above. It is checked here, and a host that is not allowed is closed.
+    if (typeof context.routeWebSocket === 'function') {
+      await context.routeWebSocket(/.*/, (ws) => {
+        const url = ws.url();
+        if (!allowedUrl(url.replace(/^ws/, 'http'))) { stop(url, refusal); ws.close({ code: 1008, reason: 'refused by guards.hosts' }); return; }
+        ws.connectToServer();
+      });
+    }
+    contexts.set(key, entry);
+    return entry;
+  }
+
+  const redactUrl = (url) => (redactor ? redactor.text(url) : url);
+
+  async function pageOf(entry) {
+    if (entry.page && !entry.page.isClosed()) return entry.page;
+    const page = await entry.context.newPage();
+    page.setDefaultTimeout(timeoutMs);
+    page.on('console', (message) => { if (message.type() === 'error' && !stopped.has(message.location()?.url)) event(entry.persona, 'console-error', { text: normal(message.text()).slice(0, 500), at: redactUrl(message.location()?.url ?? '') }); });
+    page.on('pageerror', (error) => event(entry.persona, 'page-error', { text: normal(error?.message ?? error).slice(0, 500) }));
+    // A request failed at the network. A request that the guards stopped is not counted here.
+    page.on('requestfailed', (request) => { if (!stopped.has(request.url())) event(entry.persona, 'request-failed', { url: redactUrl(request.url()), why: request.failure()?.errorText ?? 'failed' }); });
+    // An HTTP error answer (4xx, 5xx) is listed apart. It is no network failure.
+    page.on('response', (response) => { if (response.status() >= 400) event(entry.persona, 'http-error', { url: redactUrl(response.url()), status: response.status() }); });
+    entry.page = page;
+    return page;
+  }
+
+  const answer = (status, body, headers = {}) => ({ status, body, isError: typeof status === 'number' && status >= 400, headers, raw: typeof body === 'string' ? body : JSON.stringify(body) });
+
+  // A picture cannot be redacted by value, so each part of the page that shows a secret is masked.
+  async function shoot(entry, name, { full = true } = {}) {
+    const page = entry.page;
+    if (!page || page.isClosed()) throw new Error(`there is no open page to take a screenshot of "${name}"`);
+    if (typeof attach !== 'function') throw new Error('this driver was made with no place to keep screenshots');
+    const safe = String(name).replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'screenshot';
+    const path = attach(`${safe}.png`);
+    const secrets = redactor ? [...redactor.values] : [];
+    const mask = secrets.map((value) => page.getByText(value));
+    // An input shows its value, which is no text node. Mark each input whose value holds a secret, and mask it.
+    if (secrets.length) {
+      await page.evaluate((values) => { for (const el of document.querySelectorAll('input, textarea, select')) if (values.some((v) => String(el.value ?? '').includes(v))) el.setAttribute('data-atlas-mask', '1'); }, secrets);
+      mask.push(page.locator('[data-atlas-mask]'));
+    }
+    let matched = 0;
+    for (const one of mask) matched += await one.count();
+    await page.screenshot({ path, fullPage: full, mask });
+    const bytes = readFileSync(path);
+    return { name: `${safe}.png`, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex'), masked: matched };
+  }
+
+  async function call(request = {}, { credential = null, headers = {}, persona = null } = {}) {
+    if (closed) throw new Error(`the browser of "${way}" is closed`);
+    // The address is checked before any browser starts.
+    let target = null;
+    if (request.open !== undefined) {
+      target = /^[a-z][a-z0-9+.-]*:/i.test(String(request.open)) ? new URL(request.open) : new URL(`${base.replace(/\/+$/, '')}/${String(request.open).replace(/^\/+/, '')}`);
+      requireAllowed(target.href, hosts, `open ${target.pathname}`);
+    } else if (request.method) {
+      target = request.url ? new URL(request.url) : new URL(`${base.replace(/\/+$/, '')}/${String(request.path ?? '').replace(/^\/+/, '')}`);
+      requireAllowed(target.href, hosts, `${request.method} ${target.pathname}`);
+    }
+    const entry = await contextOf(persona, { headers, credential });
+    const log = (shape, response) => record?.({ way, persona, request: shape, response });
+
+    if (request.open !== undefined) {
+      const page = await pageOf(entry);
+      let response;
+      try {
+        response = await page.goto(target.href, { waitUntil: 'load', timeout: NAVIGATION_TIMEOUT });
+      } catch (error) {
+        const first = String(error?.message ?? error).split('\n')[0];
+        // The route stopped the address or a redirect of it. That is a guard refusal, so the result is `blocked`.
+        // The page is left in an error state, so it is closed and the next open makes a new one.
+        if (/ERR_BLOCKED_BY_CLIENT/.test(first)) {
+          try { await page.close(); } catch { /* it is gone */ }
+          entry.page = null;
+          throw new Blocked(`open ${target.pathname}: a request of the page, or a redirect, led to an address that guards.hosts does not allow, so the guards stopped it`);
+        }
+        throw new Error(`open ${target.pathname}: ${first}`);
+      }
+      const landed = page.url();
+      if (!INTERNAL.test(landed)) requireAllowed(landed, hosts, `the page ${target.pathname} ended on`);
+      const status = response?.status() ?? 0;
+      const out = answer(status, { url: redactUrl(landed), title: await page.title() }, response?.headers() ?? {});
+      log({ open: redactUrl(target.href) }, { status, body: out.body });
+      return out;
+    }
+
+    if (request.read !== undefined) {
+      const page = entry.page;
+      if (!page || page.isClosed()) throw new Error('read: there is no open page; open one first');
+      const wait = request.optional ? 0 : (request.timeout ?? timeoutMs);
+      const locator = locate(page, request.read);
+      let found = true;
+      if (wait > 0) {
+        try { await locator.first().waitFor({ state: 'visible', timeout: wait }); } catch { found = false; }
+      }
+      const texts = (await locator.allTextContents()).map(normal);
+      const count = texts.length;
+      found = count > 0 && (await locator.first().isVisible());
+      const body = { found, count, texts };
+      log({ read: show(request.read) }, { status: 200, body: clip(body, redactor) });
+      return answer(200, body);
+    }
+
+    if (request.click !== undefined || request.fill !== undefined) {
+      const page = entry.page;
+      if (!page || page.isClosed()) throw new Error('there is no open page; open one first');
+      const finder = request.click ?? request.fill;
+      if (request.fill !== undefined && redactor && [...redactor.values].some((v) => String(request.value ?? '').includes(v))) {
+        throw new Blocked('a fill with the value of a run secret was refused: a secret must not be typed into a page, because the page could show it or send it on');
+      }
+      const locator = locate(page, finder).first();
+      if (request.fill !== undefined) await locator.fill(String(request.value ?? ''));
+      else await locator.click();
+      // The value that was typed can be a secret: the record holds a redacted copy.
+      log(request.fill !== undefined ? { fill: show(finder), value: '[typed]' } : { click: show(finder) }, { status: 200 });
+      return answer(200, { done: true });
+    }
+
+    if (request.screenshot !== undefined) {
+      const info = await shoot(entry, request.screenshot);
+      log({ screenshot: request.screenshot }, { status: 200, body: info });
+      return answer(200, info);
+    }
+
+    if (request.method) {
+      const sendHeaders = { ...(request.headers ?? {}) };
+      const response = await entry.context.request.fetch(target.href, {
+        method: request.method.toUpperCase(), headers: sendHeaders, data: request.body, maxRedirects: 0, failOnStatusCode: false,
+      });
+      const text = await response.text();
+      let body = text;
+      if (/json/.test(response.headers()['content-type'] ?? '') && text) { try { body = JSON.parse(text); } catch { /* keep the text */ } }
+      const out = answer(response.status(), body, headersOf(response));
+      out.raw = text;
+      log({ method: request.method.toUpperCase(), url: redactUrl(target.href) }, { status: out.status, body: clip(body, redactor) });
+      return out;
+    }
+
+    throw new Error('the browser driver was called with no open, read, click, fill, screenshot or method');
+  }
+
+  // Playwright joins the values of a repeated header with a newline. A cookie reader wants the first.
+  function headersOf(response) {
+    const out = {};
+    for (const { name, value } of response.headersArray()) {
+      const key = name.toLowerCase();
+      out[key] = out[key] === undefined ? value : `${out[key]}, ${value}`;
+    }
+    return out;
+  }
+
+  // A picture of every open page, for a run that failed. It never throws.
+  async function keepOnFail(label = 'fail') {
+    const kept = [];
+    for (const entry of contexts.values()) {
+      try {
+        if (!entry.page || entry.page.isClosed()) continue;
+        const info = await shoot(entry, `${label}-${way}-${entry.persona ?? 'page'}`);
+        record?.({ way, persona: entry.persona, request: { event: 'screenshot-on-fail' }, response: info });
+        kept.push(info);
+      } catch { /* the failed run is the news; a missing picture is not */ }
+    }
+    return kept;
+  }
+
+  async function close() {
+    if (closed) return;
+    closed = true;
+    for (const entry of contexts.values()) { try { await entry.context.close(); } catch { /* the stop goes on */ } }
+    contexts.clear();
+    if (browser) { try { await browser.close(); } catch { /* the stop goes on */ } browser = null; }
+  }
+
+  return { name: 'browser', way, base, call, keepOnFail, close };
+}

package/tests/drivers/index.mjs

@@ -1,6 +1,8 @@
 // Builds the driver for a way in from its entry in tests.yaml. A driver that
-// the kit does not ship (a browser, a command) comes from the adapter:
-//   adapter.drivers = { browser: async (ctx) => driver }
+// the kit does not ship (a command, a queue) comes from the adapter:
+//   adapter.drivers = { command: async (ctx) => driver }
+// The `browser` driver ships. It needs the package "playwright", which is an optional
+// peer: without it a scenario that uses a browser way in reads `blocked`.
 // A driver has the shape { name, way, call(request, opts), close() }.
 import { isAbsolute, resolve } from 'node:path';
 import { Blocked } from '../errors.mjs';
@@ -8,10 +10,13 @@ import { requireAddressesAllowed, substitute } from '../guards.mjs';
 import { createHttpDriver } from './http.mjs';
 import { createMcpStdioDriver } from './mcp-stdio.mjs';
 import { createFunctionDriver } from './function.mjs';
+import { createBrowserDriver } from './browser.mjs';
 
-export { createHttpDriver, createMcpStdioDriver, createFunctionDriver };
+export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, createBrowserDriver };
 
-// ctx: { way, spec, env, root, hosts, redactor, record, adapter, runId }
+// ctx: { way, spec, env, root, hosts, redactor, record, adapter, runId, attach }
+// `attach(fileName)` gives the path of a file to keep as evidence (a screenshot).
+// `adapter.playwright` is the Playwright module, for a test that has none installed.
 // In a fault run, ATLAS_PRODUCT_ROOT names a throwaway copy. A process that a
 // way in starts (an MCP server) and the module of the function driver load from
 // that copy, so a planted fault is what runs. tests.yaml and the kit stay in the
@@ -41,6 +46,8 @@ export async function makeDriver(ctx) {
   switch (spec.driver) {
     case 'http':
       return createHttpDriver({ way, base: fill(spec.base), hosts, record, redactor });
+    case 'browser':
+      return createBrowserDriver({ way, base: fill(spec.base), hosts, record, redactor, attach: ctx.attach, playwright: adapter.playwright });
     case 'mcp-stdio':
       return createMcpStdioDriver({
         way, command: commandFor(fill(spec.command), root, productRoot), cwd: spec.cwd ? (isAbsolute(spec.cwd) ? spec.cwd : resolve(productRoot, spec.cwd)) : productRoot, hosts, redactor, record,

package/tests/environment.mjs

@@ -23,7 +23,7 @@ import { parseLimit } from './wait.mjs';
 
 const LIBRARY_HEADERS = { [LIBRARY_HEADER]: '1' };
 
-function freePort() {
+function onePort() {
   return new Promise((resolve, reject) => {
     const server = createServer();
     server.once('error', reject);
@@ -31,6 +31,21 @@ function freePort() {
   });
 }
 
+// A free port that no stand-in of this run uses, and that the environment
+// does not name. A product with a fixed port has not started yet when a
+// stand-in takes its port, and a stand-in can listen on another address family.
+export async function freePort(taken = []) {
+  for (let tries = 0; tries < 20; tries += 1) {
+    const port = await onePort();
+    if (!taken.includes(port)) return port;
+  }
+  throw new Blocked('no free port was found that differs from the ports the run already uses');
+}
+
+const portsOf = (standIns) => Object.values(standIns).map((one) => Number(new URL(one.url).port)).filter(Boolean);
+export const namedPorts = (spec) => [...JSON.stringify(spec ?? {}).matchAll(/(?:127\.0\.0\.1|localhost|\[::1\]):(\d{1,5})/g)].map((m) => Number(m[1]))
+  .concat(Object.values(spec?.vars ?? {}).filter((v) => /^\d{2,5}$/.test(String(v))).map(Number));
+
 // ---------------------------------------------------------------- ready
 
 async function pollReady(spec, ctx, { label, process: proc, hosts, redactor, standInOrigins = [] }) {
@@ -144,7 +159,7 @@ export async function startEnvironment({ root, tests, name, runId, redactor, ada
         continue;
       }
       if (!def?.start) throw new Blocked(`${label} has no "start" command and the adapter does not make it`);
-      const port = await freePort();
+      const port = await freePort([...namedPorts(spec), ...portsOf(standIns)]);
       const url = `http://127.0.0.1:${port}`;
       const ctx = { id: runId, port, url };
       const proc = spawnManaged({
@@ -162,7 +177,7 @@ export async function startEnvironment({ root, tests, name, runId, redactor, ada
     // 2. The vars: run secrets are new for each placeholder; references resolve in order.
     const raw = spec.vars ?? {};
     // The product gets a free port too: `{port}` in start, ready, url and vars.
-    const productPort = await freePort();
+    const productPort = await freePort([...namedPorts(spec), ...portsOf(standIns)]);
     const ctx = {
       id: runId, varsFile, port: productPort, url: spec.url ? substitute(spec.url, { id: runId, port: productPort }) : undefined,
       secret: () => { const s = newSecret(); secretsMade.push(s); redactor.add(s); return s; },

package/tests/evidence.mjs

@@ -4,7 +4,7 @@
 //     "evidence": 1, "run_id": "...", "environment": "local",
 //     "scenario": "week-conflict", "title": "...", "source": "scenarios/week-conflict.md",
 //     "source_sha256": "...", "fingerprint_method": "fp1",
-//     "started": "...", "ended": null, "verdict": "not finished",
+//     "commit": "<sha or null>", "started": "...", "ended": null, "verdict": "not finished",
 //     "ways": { "mcp": { "verdict": "pass", "reason": null, "started": "...", "ended": "..." } },
 //     "assertions": [ { "id": "week-conflict/e3#09f598", "way": "mcp", "result": "pass", "proof": { ... } } ],
 //     "tables": { "T1": "<fingerprint>" }, "fresh": [ { "rule": "...", "value": "..." } ],
@@ -18,13 +18,32 @@
 // (`<scenario>.2.json`). A file is claimed with an exclusive create, so no
 // earlier run is overwritten. Every string is redacted by value before it is
 // written.
-import { closeSync, mkdirSync, openSync, renameSync, writeFileSync } from 'node:fs';
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { join, relative } from 'node:path';
+import { spawnSync } from 'node:child_process';
 
 export const EVIDENCE_SCHEMA = 1;
 
-// The results of one assertion for one way in (design section 6.11).
-export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here']);
+// The commit under test: the head of the pull request when the environment gives it (GITHUB_HEAD_SHA, or
+// pull_request.head.sha in the event file), else GITHUB_SHA, else the HEAD of the repo, when git can say.
+function commitOf(root) {
+  if (process.env.GITHUB_HEAD_SHA) return process.env.GITHUB_HEAD_SHA;
+  if (process.env.GITHUB_EVENT_PATH) {
+    try {
+      const sha = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8'))?.pull_request?.head?.sha;
+      if (sha) return sha;
+    } catch { /* no event file that reads */ }
+  }
+  if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA;
+  try {
+    const r = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' });
+    return r.status === 0 ? r.stdout.trim() : null;
+  } catch { return null; }
+}
+
+// The results of one assertion for one way in (design section 6.11). `flaky` is never set by a
+// test. `atlas tests verdict` and the proof table read it from two attempts (verdict.mjs).
+export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here', 'flaky']);
 
 const MAX_BODY = 8000;
 
@@ -47,13 +66,17 @@ export class Evidence {
       scenario: scenario.id, title: scenario.title,
       source: relative(root, scenario.file).split('\\').join('/'), source_sha256: scenario.sha256,
       fingerprint_method: scenario.method,
-      started: new Date().toISOString(), ended: null, verdict: 'not finished',
+      commit: commitOf(root), started: new Date().toISOString(), ended: null, verdict: 'not finished',
       ways: {}, assertions: [], tables: {}, fresh: [], calls: [], notes: [], files: [],
       provided_secrets: providedSecrets, fault_run: process.env.ATLAS_FAULT_RUN || null,
     };
-    for (const way of scenario.through) {
+    // One entry for each assertion and each way in that it belongs to (scenario.mjs, waysOf). An
+    // assertion that reads a result of a step with its own way in (the page) has that way only.
+    for (const way of scenario.ways ?? scenario.through) {
       this.record.ways[way] = { verdict: 'not finished', reason: null, started: null, ended: null };
-      for (const a of scenario.assertions) this.record.assertions.push({ id: a.fullId, way, result: 'not checked', proof: null });
+    }
+    for (const a of scenario.assertions) {
+      for (const way of a.ways ?? scenario.through) this.record.assertions.push({ id: a.fullId, way, result: 'not checked', proof: null });
     }
     mkdirSync(this.dir, { recursive: true });
     this.path = this.claim(scenario.id);

package/tests/index.mjs

@@ -2,10 +2,11 @@
 export { scenario, defineAdapter, waitUntil, RUN_ID } from './contract.mjs';
 export { startEnvironment, parseDotenv, provideSecrets } from './environment.mjs';
 export { createStandIn, serveStandIn, CALLS_PATH, LIBRARY_HEADER } from './stand-in.mjs';
-export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, makeDriver } from './drivers/index.mjs';
+export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, createBrowserDriver, makeDriver } from './drivers/index.mjs';
 export { parseTestsYaml, readTestsYaml, checkTestsYaml, guardParts, compareGuardParts, GUARD_PARTS, looksLikeSecret } from './tests-yaml.mjs';
 export { parseScenario, readScenario, addFingerprints, FINGERPRINT_METHOD } from './scenario.mjs';
 export { Blocked, Unavailable, WaitFailed, WaitTimeout, GateFailed } from './errors.mjs';
 export { Redactor } from './redact.mjs';
 export { Evidence, EVIDENCE_SCHEMA, RESULTS } from './evidence.mjs';
 export { checkArea } from './link-check.mjs';
+export { listNamed, decide, settleRecords, readAttempts, loadAreaScenarios, keyOf } from './verdict.mjs';

package/tests/link-check.mjs

@@ -466,6 +466,25 @@ export function newestRunPerScenario(evidenceDir) {
   return new Map([...best].map(([scenario, b]) => [scenario, b.name]));
 }
 
+// The newest run of the whole folder, by the start time of its evidence (a name breaks a tie).
+export function newestRunId(evidenceDir) {
+  let best = null;
+  if (!existsSync(evidenceDir)) return null;
+  for (const name of readdirSync(evidenceDir)) {
+    const dir = join(evidenceDir, name);
+    if (!statSync(dir).isDirectory()) continue;
+    for (const file of readdirSync(dir)) {
+      if (!file.endsWith('.json')) continue;
+      let data;
+      try { data = JSON.parse(readFileSync(join(dir, file), 'utf8')); } catch { continue; }
+      if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+      const started = String(data.started);
+      if (!best || started > best.started || (started === best.started && name > best.name)) best = { started, name };
+    }
+  }
+  return best ? best.name : null;
+}
+
 // The run half reads the files of one run only, and only evidence of the
 // scenario file as it is now (source_sha256).
 export function checkRun({ evidenceDir, runId = null, environment = null, scenarios, rel, findings }) {
@@ -488,8 +507,8 @@ export function checkRun({ evidenceDir, runId = null, environment = null, scenar
       runOf.set(data.scenario, id);
     }
   }
-  const reached = new Set(['pass', 'fail', 'not exercised', 'not here']);
-  const ran = new Set(['pass', 'fail', 'not exercised']);
+  const reached = new Set(['pass', 'fail', 'not exercised', 'not here', 'flaky']);
+  const ran = new Set(['pass', 'fail', 'not exercised', 'flaky']);
   for (const s of scenarios) {
     if (!s.id) continue;
     const all = records.filter((r) => r.data.scenario === s.id);
@@ -500,7 +519,7 @@ export function checkRun({ evidenceDir, runId = null, environment = null, scenar
       continue;
     }
     for (const a of s.assertions) {
-      for (const way of s.through) {
+      for (const way of a.ways ?? s.through) {
         const seen = mine.some((r) => r.data.assertions?.some((e) => e.id === a.fullId && e.way === way && reached.has(e.result)));
         if (!seen) findings.push({ file: rel(s.file), line: a.line, code: 'not-checked', message: `no run checked ${s.id}/${a.id}#${a.fp} through ${way}` });
       }

package/tests/scenario.mjs

@@ -271,6 +271,23 @@ export function readScenario(path) {
 
 const runsIn = (scenario) => [].concat(scenario.fields['runs-in'] ?? []).map(String);
 
+// The ways in that an assertion is checked through. An assertion reads a kept result. When the
+// step that kept it names its own way in ("As `owner`, through `page`, ..."), the assertion belongs
+// to that way: the proof table shows it under that column (design section 6.3). An assertion that
+// names no kept result of such a step belongs to every way of `through:`. The ways of `through:`
+// stay first. The words and the fingerprint do not change.
+function waysOf(assertion, scenario, kept) {
+  const named = new Set();
+  let plain = false;
+  for (const name of assertion.names) {
+    const step = kept.get(name);
+    if (!step) continue;
+    if (step.way) named.add(step.way); else plain = true;
+  }
+  if (!named.size || plain) return [...new Set([...scenario.through, ...named])];
+  return [...named];
+}
+
 // Sets `fp` and `fullId` on every assertion. `tests` is the parsed value of
 // tests.yaml. Returns the scenario.
 export function addFingerprints(scenario, tests) {
@@ -315,7 +332,9 @@ export function addFingerprints(scenario, tests) {
     lines.push(`through=${[...scenario.through].sort().join(',')}`, `runs-in=${runsIn(scenario).sort().join(',')}`);
     a.fp = fingerprintOf([a.words, ...lines].join('\n'));
     a.fullId = `${scenario.id}/${a.id}#${a.fp}`;
+    a.ways = waysOf(a, scenario, kept);
   }
+  scenario.ways = [...new Set([...scenario.through, ...scenario.assertions.flatMap((a) => a.ways)])];
   return scenario;
 }