@arjunkhera/atlas 0.3.7 → 0.3.9

package/tests/drivers/mcp-stdio.mjs

@@ -0,0 +1,174 @@
+// The mcp-stdio driver: a small JSON-RPC 2.0 client for an MCP server that
+// speaks over standard input and output, one JSON message on each line (the
+// MCP stdio transport). Atlas has no dependency, so it does not use the
+// official client library. It does what the tests need: initialize, the
+// initialized notification, tools/list and tools/call.
+//
+// The seven duties of a driver (design section 7.2):
+//   1. Before it starts the server, it checks every address in the env that it
+//      gives the server against guards.hosts. A server that gets no address is
+//      refused, because it could fall back to a real one. The result is `blocked`.
+//   2. A credential goes only into the env of the server it belongs to. Each
+//      distinct env starts its own server process.
+//   3. It records each tool call and each answer.
+//   4. Recorded text has run secrets and named secrets removed, by value.
+//   5. It returns the raw answer: { status, body, isError, raw }.
+//      `body` is the result of tools/call, or the JSON-RPC error. `raw` is the whole message.
+//      Reading the product's error form is the work of the actions.
+//   6. It signs in as a persona: the env of the spec may hold {persona.credential}.
+//   7. It stops each server it started, on close and when the process ends.
+import { Blocked, Unavailable } from '../errors.mjs';
+import { childEnv, requireLocalAddresses } from '../guards.mjs';
+import { spawnManaged } from '../procs.mjs';
+import { clip } from '../evidence.mjs';
+
+const PROTOCOL = '2025-06-18';
+// The most that one line of the server may hold. A longer line is refused.
+export const MAX_LINE_BYTES = 4 * 1024 * 1024;
+
+class Session {
+  constructor({ proc, timeoutMs }) {
+    this.proc = proc;
+    this.timeoutMs = timeoutMs;
+    this.nextId = 1;
+    this.pending = new Map();
+    this.buffer = '';
+    this.closed = false;
+    proc.child.stdout.on('data', (chunk) => this.onData(chunk.toString('utf8')));
+    proc.child.stdin.on('error', () => { /* the server stopped; the exit handler reports it */ });
+    proc.exited.then(({ code, signal }) => {
+      this.closed = true;
+      for (const { reject } of this.pending.values()) reject(new Unavailable(`the MCP server stopped (${code ?? signal}). Last output: ${proc.tail().slice(-400)}`));
+      this.pending.clear();
+    });
+  }
+
+  overflow() {
+    this.buffer = '';
+    this.closed = true;
+    const why = new Unavailable(`the MCP server sent a line over ${MAX_LINE_BYTES / (1024 * 1024)} MB; the line was refused and the server was stopped`);
+    for (const { reject, timer } of this.pending.values()) { clearTimeout(timer); reject(why); }
+    this.pending.clear();
+    this.proc.stop();
+  }
+
+  onData(text) {
+    if (this.closed) return;
+    this.buffer += text;
+    let at;
+    while ((at = this.buffer.indexOf('\n')) >= 0) {
+      if (at > MAX_LINE_BYTES) return this.overflow();
+      const line = this.buffer.slice(0, at).trim();
+      this.buffer = this.buffer.slice(at + 1);
+      if (!line) continue;
+      let message;
+      try { message = JSON.parse(line); } catch { continue; }
+      this.onMessage(message);
+    }
+    if (this.buffer.length > MAX_LINE_BYTES) this.overflow();
+  }
+
+  onMessage(message) {
+    if (message.id !== undefined && (message.result !== undefined || message.error !== undefined)) {
+      const waiting = this.pending.get(message.id);
+      if (waiting) { this.pending.delete(message.id); clearTimeout(waiting.timer); waiting.resolve(message); }
+    } else if (message.id !== undefined && message.method) {
+      // The client offers nothing to a server that asks.
+      this.send({ jsonrpc: '2.0', id: message.id, error: { code: -32601, message: 'method not found' } });
+    }
+  }
+
+  send(message) {
+    if (this.closed) throw new Unavailable('the MCP server is not running');
+    this.proc.child.stdin.write(`${JSON.stringify(message)}\n`);
+  }
+
+  request(method, params) {
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new Unavailable(`the MCP server did not answer ${method} in ${this.timeoutMs} ms`));
+      }, this.timeoutMs);
+      this.pending.set(id, { resolve, reject, timer });
+      try { this.send({ jsonrpc: '2.0', id, method, params }); } catch (error) { clearTimeout(timer); this.pending.delete(id); reject(error); }
+    });
+  }
+
+  notify(method, params) { this.send({ jsonrpc: '2.0', method, params }); }
+
+  async open() {
+    // A server that stops or stays mute before it answers initialize never started: `blocked`.
+    let init;
+    try { init = await this.request('initialize', { protocolVersion: PROTOCOL, capabilities: {}, clientInfo: { name: 'atlas-tests', version: '1' } }); } catch (error) {
+      if (error instanceof Unavailable) throw new Blocked(`the MCP server did not start: ${error.message}`);
+      throw error;
+    }
+    if (init.error) throw new Blocked(`the MCP server refused initialize: ${init.error.message}`);
+    this.notify('notifications/initialized');
+    return init.result;
+  }
+
+  async close() {
+    this.closed = true;
+    try { this.proc.child.stdin.end(); } catch { /* it is closed */ }
+    await this.proc.stop();
+  }
+}
+
+export function createMcpStdioDriver({ way, command, env: specEnv = {}, cwd, hosts, redactor, record: rawRecord, timeoutMs = 30000 }) {
+  const record = rawRecord && ((call) => rawRecord(redactor ? redactor.deep(call) : call));
+  if (!command) throw new Error(`the way in "${way}" has no command`);
+  const sessions = new Map();
+
+  const fill = (credential) => Object.fromEntries(Object.entries(specEnv).map(([k, v]) => [k, String(v).replaceAll('{persona.credential}', credential ?? '')]));
+
+  async function sessionFor(credential) {
+    const given = fill(credential);
+    const key = JSON.stringify(given);
+    if (sessions.has(key)) return sessions.get(key);
+    // Duty 1: check the address before the server starts.
+    // The check runs on the whole env the server gets, not only on the part the way in sets.
+    const full = childEnv(process.env, given);
+    requireLocalAddresses(given, hosts, `the MCP server of "${way}"`, full);
+    const proc = spawnManaged({ command, cwd, redactor, stdin: true, env: full });
+    const session = new Session({ proc, timeoutMs });
+    sessions.set(key, session);
+    try { await session.open(); } catch (error) { sessions.delete(key); await session.close(); throw error; }
+    return session;
+  }
+
+  // request: { tool, arguments } for tools/call, or { method, params } for any other call.
+  async function call(request = {}, { credential = null, persona = null } = {}) {
+    const session = await sessionFor(credential);
+    const method = request.tool ? 'tools/call' : request.method;
+    const params = request.tool ? { name: request.tool, arguments: request.arguments ?? {} } : request.params;
+    if (!method) throw new Error('an MCP request names a tool, or a method');
+    const message = await session.request(method, params);
+    const failed = message.error !== undefined;
+    const body = failed ? message.error : message.result;
+    const answer = {
+      status: failed ? message.error.code : (body?.isError ? 'tool-error' : 'ok'),
+      body, isError: failed || body?.isError === true, raw: message,
+    };
+    record?.({
+      way, persona,
+      request: { method, params: clip(params, redactor) },
+      response: { status: answer.status, body: clip(body, redactor) },
+    });
+    return answer;
+  }
+
+  async function listTools(opts) {
+    const answer = await call({ method: 'tools/list', params: {} }, opts);
+    return answer.body?.tools ?? [];
+  }
+
+  async function close() {
+    const all = [...sessions.values()];
+    sessions.clear();
+    await Promise.all(all.map((s) => s.close()));
+  }
+
+  return { name: 'mcp-stdio', way, call, listTools, close };
+}

package/tests/environment.mjs

@@ -0,0 +1,227 @@
+// The run environment: start the stand-ins, then the product; make the run
+// secrets and the vars file new for each run; wait until the product is
+// ready; check that the product called each stand-in; stop everything.
+//
+//   const env = await startEnvironment({ root, tests, name: 'local', runId, redactor, adapter });
+//   env.url, env.vars, env.standIns.engram.url
+//   await env.stop();
+//
+// Every child process starts from an allow-list environment (guards.mjs,
+// childEnv), and each resolved var is set as a real env value on the product.
+// The vars file lives in a new temporary folder and is removed on stop.
+import { mkdtempSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { dirname, join, parse } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { spawnManaged } from './procs.mjs';
+import { Blocked } from './errors.mjs';
+import { resolveEnvironments } from './tests-yaml.mjs';
+import { childEnv, hostsOf, newSecret, requireAddressesAllowed, requireAllowed, substitute } from './guards.mjs';
+import { CALLS_PATH, countCalls, LIBRARY_HEADER } from './stand-in.mjs';
+import { parseLimit } from './wait.mjs';
+
+const LIBRARY_HEADERS = { [LIBRARY_HEADER]: '1' };
+
+function freePort() {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => { const { port } = server.address(); server.close(() => resolve(port)); });
+  });
+}
+
+// ---------------------------------------------------------------- ready
+
+async function pollReady(spec, ctx, { label, process: proc, hosts, redactor, standInOrigins = [] }) {
+  const url = substitute(spec.get ?? '', ctx);
+  if (!url) throw new Blocked(`${label}: "ready" has no "get" address`);
+  const parsed = requireAllowed(url, hosts, `${label} ready check`);
+  // A ready address must not point at a stand-in: a poll there could stand in
+  // for the first call of the product.
+  if (standInOrigins.includes(parsed.origin)) throw new Blocked(`${label}: the "ready" address ${parsed.origin} is a stand-in; the ready check must ask the product itself`);
+  const want = spec.status ?? 200;
+  const limit = parseLimit(spec.limit ?? '30s');
+  const deadline = Date.now() + limit;
+  let last = 'no answer';
+  for (;;) {
+    if (proc && (proc.child.exitCode !== null || proc.child.signalCode !== null)) {
+      throw new Blocked(`${label} stopped before it was ready (${proc.child.exitCode ?? proc.child.signalCode}). Last output: ${proc.tail().slice(-600)}`);
+    }
+    try {
+      const res = await fetch(url, { redirect: 'manual', headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(2000) });
+      await res.arrayBuffer();
+      if (res.status === want) return;
+      last = `status ${res.status}`;
+    } catch (error) {
+      last = error.cause?.code ?? error.name;
+    }
+    if (Date.now() >= deadline) {
+      throw new Blocked(`${label} was not ready after ${spec.limit ?? '30s'} (${last}). Last output: ${redactor?.text(proc?.tail() ?? '').slice(-600)}`);
+    }
+    await sleep(150);
+  }
+}
+
+// ---------------------------------------------------------------- secrets
+
+// KEY=value lines of a dotenv file. No expansion.
+export function parseDotenv(text) {
+  const out = {};
+  for (const line of String(text).split('\n')) {
+    const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/.exec(line);
+    if (!m || line.trim().startsWith('#')) continue;
+    let value = m[2];
+    if (/^(["']).*\1$/.test(value)) value = value.slice(1, -1);
+    out[m[1]] = value;
+  }
+  return out;
+}
+
+function findDotenv(from) {
+  let dir = from;
+  for (;;) {
+    const file = join(dir, '.env.test');
+    if (existsSync(file)) return file;
+    if (existsSync(join(dir, '.git')) || dir === parse(dir).root) return null;
+    dir = dirname(dir);
+  }
+}
+
+// The value of each provided secret that the scenario names. On the Mac it is
+// in .env.test at the repo root. In CI it is a CI secret of the same name.
+// With no value, the result is `blocked` and names the key.
+export function provideSecrets(names, { root, tests, env = process.env }) {
+  const out = {};
+  if (!names.length) return out;
+  const file = findDotenv(root);
+  const fromFile = file ? parseDotenv(readFileSync(file, 'utf8')) : {};
+  for (const name of names) {
+    if (!Object.hasOwn(tests.secrets ?? {}, name)) continue;
+    const value = fromFile[name] ?? env[name];
+    if (!value) throw new Blocked(`the scenario names the key ${name} and no value exists: add it to .env.test at the repo root, or set the CI secret of the same name`);
+    out[name] = value;
+  }
+  return out;
+}
+
+const dotenvLine = (name, value) => `${name}=${/^[A-Za-z0-9_./:@%+=,-]*$/.test(value) ? value : JSON.stringify(value)}\n`;
+
+// ---------------------------------------------------------------- start
+
+export async function startEnvironment({ root, tests, name, runId, redactor, adapter = {}, provided = {} }) {
+  const environments = resolveEnvironments(tests.environments ?? {});
+  const spec = environments[name];
+  if (!spec) throw new Blocked(`tests.yaml has no environment "${name}"`);
+  if (spec.allowed === false) throw new Blocked(`the environment "${name}" is not allowed: "allowed" is false until a test environment and a scoped key exist`);
+  // A deployed environment is closed until the owner opens it with `allowed: true`.
+  if (name === 'deployed' && spec.allowed !== true) throw new Blocked(`the environment "deployed" is closed: it has no "allowed: true". Only the owner opens it, with a test environment and a scoped key`);
+  const hosts = hostsOf(tests);
+  const workDir = mkdtempSync(join(tmpdir(), 'atlas-run-'));
+  const varsFile = join(workDir, 'vars.env');
+  const managed = [];
+  const stoppers = [];
+  const standIns = {};
+  const resolvedVars = {};
+  const secretsMade = [];
+  for (const value of Object.values(provided)) redactor.add(value);
+
+  const stop = async () => {
+    for (const fn of stoppers.reverse()) { try { await fn(); } catch { /* the stop goes on */ } }
+    for (const proc of managed.reverse()) await proc.stop(proc.stopCommand);
+    rmSync(workDir, { recursive: true, force: true });
+  };
+
+  try {
+    // 1. The stand-ins.
+    for (const [standInName, def] of Object.entries(tests['stand-ins'] ?? {})) {
+      const label = `stand-in "${standInName}"`;
+      if (adapter.standIns?.[standInName]) {
+        const made = await adapter.standIns[standInName]({ name: standInName, runId, hosts });
+        if (made.stop) stoppers.push(made.stop);
+        requireAllowed(made.url, hosts, label);
+        standIns[standInName] = { url: made.url, calls: made.calls ?? (async () => countCalls(await (await fetch(`${made.url}${CALLS_PATH}`, { headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(3000) })).json())) };
+        continue;
+      }
+      if (!def?.start) throw new Blocked(`${label} has no "start" command and the adapter does not make it`);
+      const port = await freePort();
+      const url = `http://127.0.0.1:${port}`;
+      const ctx = { id: runId, port, url };
+      const proc = spawnManaged({
+        command: substitute(def.start, ctx), cwd: root, redactor,
+        env: childEnv(process.env, { PORT: port, ATLAS_STAND_IN_PORT: port, ATLAS_RUN_ID: runId }),
+      });
+      managed.push(proc);
+      await pollReady(def.ready ?? { get: `${url}${CALLS_PATH}`, status: 200, limit: '30s' }, ctx, { label, process: proc, hosts, redactor, standInOrigins: Object.values(standIns).map((one) => new URL(one.url).origin) });
+      standIns[standInName] = {
+        url,
+        calls: async () => countCalls(await (await fetch(`${url}${CALLS_PATH}`, { headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(3000) })).json()),
+      };
+    }
+
+    // 2. The vars: run secrets are new for each placeholder; references resolve in order.
+    const raw = spec.vars ?? {};
+    // The product gets a free port too: `{port}` in start, ready, url and vars.
+    const productPort = await freePort();
+    const ctx = {
+      id: runId, varsFile, port: productPort, url: spec.url ? substitute(spec.url, { id: runId, port: productPort }) : undefined,
+      secret: () => { const s = newSecret(); secretsMade.push(s); redactor.add(s); return s; },
+      standIn: (n) => standIns[n]?.url,
+      vars: (n) => resolveVar(n, []),
+    };
+    function resolveVar(varName, stack) {
+      if (Object.hasOwn(resolvedVars, varName)) return resolvedVars[varName];
+      if (!Object.hasOwn(raw, varName)) throw new Blocked(`a placeholder names the variable ${varName}, which "vars" does not set`);
+      if (stack.includes(varName)) throw new Blocked(`the variables ${[...stack, varName].join(' -> ')} refer to each other in a circle`);
+      const value = substitute(String(raw[varName]), { ...ctx, vars: (n) => resolveVar(n, [...stack, varName]) });
+      resolvedVars[varName] = value;
+      return value;
+    }
+    for (const varName of Object.keys(raw)) resolveVar(varName, []);
+    for (const [varName, value] of Object.entries(resolvedVars)) {
+      if (/(TOKEN|SECRET|KEY|PASSWORD|CREDENTIAL)/i.test(varName)) redactor.add(value);
+    }
+    // Every value that is an address, of any scheme or as host:port, must be on
+    // the list, whatever the name of its variable.
+    requireAddressesAllowed(resolvedVars, hosts, 'the vars of the environment');
+    writeFileSync(varsFile, Object.entries(resolvedVars).map(([n, v]) => dotenvLine(n, v)).join(''), { mode: 0o600 });
+
+    // 3. The product. In a fault run, ATLAS_PRODUCT_ROOT names the folder of a
+    // throwaway copy: the product starts there, while tests.yaml, the support
+    // code and the stand-ins stay in the real worktree.
+    const productRoot = process.env.ATLAS_PRODUCT_ROOT || root;
+    const given = { ...provided, ATLAS_RUN_ID: runId };
+    let product = null;
+    if (spec.start) {
+      // An allow-list environment. Each resolved var is a real value that
+      // replaces any parent value of the same name; the vars file stays too.
+      product = spawnManaged({
+        command: substitute(spec.start, ctx), cwd: productRoot, redactor,
+        env: childEnv(process.env, { ...resolvedVars, ...given }),
+      });
+      product.stopCommand = spec.stop;
+      managed.push(product);
+    }
+    if (spec.ready) await pollReady(spec.ready, ctx, { label: `the product (${name})`, process: product, hosts, redactor, standInOrigins: Object.values(standIns).map((one) => new URL(one.url).origin) });
+    if (spec.url) requireAllowed(substitute(spec.url, ctx), hosts, 'the environment url');
+
+    const env = {
+      name, url: spec.url ? substitute(spec.url, ctx) : null, port: productPort, vars: { ...resolvedVars }, varsFile, hosts, standIns, root, productRoot,
+      provided: Object.keys(provided), output: () => product?.tail() ?? '', stop,
+    };
+
+    // 4. The first-call rule: the product must have called each stand-in.
+    await adapter.warmUp?.(env);
+    for (const [standInName, one] of Object.entries(standIns)) {
+      let count = 0;
+      try { count = await one.calls(); } catch (error) {
+        throw new Blocked(`stand-in "${standInName}" does not answer ${CALLS_PATH}: ${error.cause?.code ?? error.message}`);
+      }
+      if (count < 1) throw new Blocked(`stand-in "${standInName}" saw no first call from the product before step 1: the product may talk to something else`);
+    }
+    return env;
+  } catch (error) {
+    await stop();
+    throw error;
+  }
+}

package/tests/errors.mjs

@@ -0,0 +1,27 @@
+// The kinds of "could not run" and the other stops. Each one has one result.
+
+// A setup failed, a guard refused, or a precondition does not hold. The
+// result is `blocked`, never `fail`.
+export class Blocked extends Error {
+  constructor(reason) { super(reason); this.name = 'Blocked'; }
+}
+
+// A service that the test needs does not answer. The result is `blocked`.
+export class Unavailable extends Error {
+  constructor(reason) { super(reason); this.name = 'Unavailable'; }
+}
+
+// A wait met its fail state. This is a product result, so it is a `fail`.
+export class WaitFailed extends Error {
+  constructor(reason, last) { super(reason); this.name = 'WaitFailed'; this.last = last; }
+}
+
+// A wait ran out of time. This is a product result, so it is a `fail`.
+export class WaitTimeout extends Error {
+  constructor(reason, last) { super(reason); this.name = 'WaitTimeout'; this.last = last; }
+}
+
+// Thrown by a failed gate. It stops the body of the test.
+export class GateFailed extends Error {
+  constructor(reason) { super(reason); this.name = 'GateFailed'; }
+}

package/tests/evidence.mjs

@@ -0,0 +1,113 @@
+// The evidence file. Schema `evidence: 1`.
+//
+//   {
+//     "evidence": 1, "run_id": "...", "environment": "local",
+//     "scenario": "week-conflict", "title": "...", "source": "scenarios/week-conflict.md",
+//     "source_sha256": "...", "fingerprint_method": "fp1",
+//     "started": "...", "ended": null, "verdict": "not finished",
+//     "ways": { "mcp": { "verdict": "pass", "reason": null, "started": "...", "ended": "..." } },
+//     "assertions": [ { "id": "week-conflict/e3#09f598", "way": "mcp", "result": "pass", "proof": { ... } } ],
+//     "tables": { "T1": "<fingerprint>" }, "fresh": [ { "rule": "...", "value": "..." } ],
+//     "calls": [ { "n": 1, "way": "mcp", "persona": "planner-1", "request": { }, "response": { } } ],
+//     "notes": [], "files": [], "provided_secrets": [], "fault_run": null
+//   }
+//
+// The file is written when the run starts and after each assertion, so a run
+// that dies leaves a record with the verdict "not finished". One file for each
+// run and scenario. A second attempt in the same run id gets a new file
+// (`<scenario>.2.json`). A file is claimed with an exclusive create, so no
+// earlier run is overwritten. Every string is redacted by value before it is
+// written.
+import { closeSync, mkdirSync, openSync, renameSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+export const EVIDENCE_SCHEMA = 1;
+
+// The results of one assertion for one way in (design section 6.11).
+export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here']);
+
+const MAX_BODY = 8000;
+
+// Redact first, then cut. A cut before the redaction could split a secret, and
+// the half that stays would no longer match its value.
+export function clip(value, redactor = null) {
+  const safe = redactor ? redactor.deep(value) : value;
+  const text = typeof safe === 'string' ? safe : JSON.stringify(safe);
+  if (text === undefined || text.length <= MAX_BODY) return safe;
+  return `${text.slice(0, MAX_BODY)}... [${text.length - MAX_BODY} more characters]`;
+}
+
+export class Evidence {
+  constructor({ evidenceDir, runId, environment, scenario, root, redactor, providedSecrets = [] }) {
+    this.dir = join(evidenceDir, runId);
+    this.root = root;
+    this.redactor = redactor;
+    this.record = {
+      evidence: EVIDENCE_SCHEMA, run_id: runId, environment,
+      scenario: scenario.id, title: scenario.title,
+      source: relative(root, scenario.file).split('\\').join('/'), source_sha256: scenario.sha256,
+      fingerprint_method: scenario.method,
+      started: new Date().toISOString(), ended: null, verdict: 'not finished',
+      ways: {}, assertions: [], tables: {}, fresh: [], calls: [], notes: [], files: [],
+      provided_secrets: providedSecrets, fault_run: process.env.ATLAS_FAULT_RUN || null,
+    };
+    for (const way of scenario.through) {
+      this.record.ways[way] = { verdict: 'not finished', reason: null, started: null, ended: null };
+      for (const a of scenario.assertions) this.record.assertions.push({ id: a.fullId, way, result: 'not checked', proof: null });
+    }
+    mkdirSync(this.dir, { recursive: true });
+    this.path = this.claim(scenario.id);
+    this.write();
+  }
+
+  claim(scenarioId) {
+    for (let n = 1; ; n += 1) {
+      const path = join(this.dir, n === 1 ? `${scenarioId}.json` : `${scenarioId}.${n}.json`);
+      try { closeSync(openSync(path, 'wx')); return path; } catch (error) { if (error.code !== 'EEXIST') throw error; }
+    }
+  }
+
+  entry(id, way) {
+    const found = this.record.assertions.find((a) => a.id === id && a.way === way);
+    if (!found) throw new Error(`the evidence has no assertion ${id} for the way in "${way}"`);
+    return found;
+  }
+
+  setResult(id, way, result, proof) {
+    if (!RESULTS.includes(result)) throw new Error(`"${result}" is not a result`);
+    const entry = this.entry(id, way);
+    entry.result = result;
+    entry.proof = proof ?? null;
+    this.write();
+  }
+
+  setWay(way, patch) {
+    Object.assign(this.record.ways[way], patch);
+    const verdicts = Object.values(this.record.ways).map((w) => w.verdict);
+    this.record.verdict = verdicts.includes('fail') ? 'fail'
+      : verdicts.includes('blocked') ? 'blocked'
+        : verdicts.every((v) => v === 'pass' || v === 'not here') ? (verdicts.every((v) => v === 'not here') ? 'not here' : 'pass') : 'not finished';
+    if (!verdicts.includes('not finished')) this.record.ended = new Date().toISOString();
+    this.write();
+  }
+
+  addCall(call) {
+    const n = this.record.calls.length + 1;
+    this.record.calls.push({ n, ...call });
+    return n;
+  }
+
+  addFile(path) {
+    this.record.files.push(relative(this.dir, path).split('\\').join('/'));
+    this.write();
+  }
+
+  note(text) { this.record.notes.push(String(text)); this.write(); }
+
+  write() {
+    const safe = this.redactor.deep(this.record);
+    const temp = `${this.path}.tmp`;
+    writeFileSync(temp, `${JSON.stringify(safe, null, 2)}\n`);
+    renameSync(temp, this.path);
+  }
+}

package/tests/fresh.mjs

@@ -0,0 +1,42 @@
+// Fresh values. Every value that a test makes carries the run id, so two runs
+// never share data, and a clean-up can find what a killed run left behind.
+// A value that cannot hold text, such as a date, is picked from the run id.
+import { createHash, randomBytes } from 'node:crypto';
+
+export function makeRunId(env = process.env) {
+  if (env.ATLAS_RUN_ID) return env.ATLAS_RUN_ID;
+  if (env.GITHUB_RUN_ID) return `ci${env.GITHUB_RUN_ID}-${env.GITHUB_RUN_ATTEMPT ?? 1}`;
+  return `run${randomBytes(3).toString('hex')}`;
+}
+
+// The first Monday after 1 January 2035, then a number of whole weeks.
+const FIRST_MONDAY = Date.UTC(2035, 0, 1) + (((8 - new Date(Date.UTC(2035, 0, 1)).getUTCDay()) % 7) || 7) * 86400000;
+const SPREAD_WEEKS = 520;
+
+export function mondayFor(runId, count) {
+  const weeks = createHash('sha256').update(`${runId}:${count}`).digest().readUInt32BE(0) % SPREAD_WEEKS;
+  return new Date(FIRST_MONDAY + weeks * 7 * 86400000).toISOString().slice(0, 10);
+}
+
+// A maker of fresh values for one run. `rules` is the `data` map of tests.yaml.
+// A rule whose name holds "monday" makes a Monday. Any other rule makes a
+// name: the rule, the run id and a count.
+export function makeFresh(runId, rules, adapterRules = {}) {
+  const counts = new Map();
+  const made = [];
+  const fresh = (rule) => {
+    if (!Object.hasOwn(rules ?? {}, rule) && !Object.hasOwn(adapterRules, rule)) {
+      throw new Error(`there is no data rule "${rule}" in tests.yaml`);
+    }
+    const count = (counts.get(rule) ?? 0) + 1;
+    counts.set(rule, count);
+    let value;
+    if (Object.hasOwn(adapterRules, rule)) value = adapterRules[rule]({ runId, count, rule });
+    else if (/monday/i.test(rule)) value = mondayFor(runId, `${rule}:${count}`);
+    else value = `${rule.replace(/^fresh-/, '')}-${runId}-${count}`;
+    made.push({ rule, value });
+    return value;
+  };
+  fresh.made = made;
+  return fresh;
+}