@arjunkhera/atlas 0.3.6 → 0.3.8

package/tests/drivers/http.mjs

@@ -0,0 +1,68 @@
+// The http driver. It calls a REST or plain HTTP route with `fetch`.
+//
+// The seven duties of a driver (design section 7.2):
+//   1. It refuses an address that guards.hosts does not allow (`blocked`).
+//   2. It sends a credential only to the origin it belongs to.
+//   3. It records each call and each answer.
+//   4. It removes run secrets and named secrets from what it records, by value.
+//   5. It returns the raw answer: { status, body, isError, headers, raw }.
+//   6. It signs in as a persona, never as the person who runs the test.
+//   7. It cleans up what it opened. This driver opens nothing.
+import { Unavailable } from '../errors.mjs';
+import { requireAllowed, sameOrigin } from '../guards.mjs';
+import { clip } from '../evidence.mjs';
+import { LIBRARY_HEADER } from '../stand-in.mjs';
+
+export function createHttpDriver({ way, base, hosts, record: rawRecord, redactor, timeoutMs = 30000 }) {
+  const record = rawRecord && ((call) => rawRecord(redactor ? redactor.deep(call) : call));
+  if (!base) throw new Error(`the way in "${way}" has no base address`);
+  const origin = new URL(base).origin;
+  requireAllowed(base, hosts, `the way in "${way}"`);
+
+  async function call(request = {}, { credential = null, headers: personaHeaders = {}, persona = null } = {}) {
+    const method = (request.method ?? 'GET').toUpperCase();
+    let target;
+    if (request.url) target = new URL(request.url);
+    else {
+      const path = String(request.path ?? '');
+      target = new URL(`${base.replace(/\/+$/, '')}${path.startsWith('/') || path === '' ? '' : '/'}${path}`);
+    }
+    for (const [k, v] of Object.entries(request.query ?? {})) target.searchParams.set(k, String(v));
+    requireAllowed(target.href, hosts, `${method} ${target.pathname}`);
+    const own = sameOrigin(target.href, origin);
+    // A call of the kit carries this header, so a stand-in does not count it as a call of the product.
+    const headers = { ...(request.headers ?? {}), [LIBRARY_HEADER]: '1' };
+    const sent = [];
+    // A credential goes only to the origin of this way in.
+    if (own) {
+      for (const [k, v] of Object.entries(personaHeaders)) { headers[k] = v; sent.push(k); }
+      if (credential) { headers['authorization'] = `Bearer ${credential}`; sent.push('authorization'); }
+    }
+    const init = { method, headers, redirect: 'manual', signal: AbortSignal.timeout(timeoutMs) };
+    if (request.body !== undefined) {
+      if (request.raw) init.body = request.body;
+      else { init.body = JSON.stringify(request.body); headers['content-type'] ??= 'application/json'; }
+    }
+    let response;
+    let text;
+    try {
+      response = await fetch(target, init);
+      text = await response.text();
+    } catch (error) {
+      throw new Unavailable(`${method} ${target.pathname}: ${error.cause?.code ?? error.name}`);
+    }
+    let body = text;
+    if (/json/.test(response.headers.get('content-type') ?? '') && text) {
+      try { body = JSON.parse(text); } catch { /* keep the text */ }
+    }
+    const answer = { status: response.status, body, isError: response.status >= 400, headers: Object.fromEntries(response.headers), raw: text };
+    record?.({
+      way, persona,
+      request: { method, url: target.href, credential_headers: sent, body: request.body === undefined ? undefined : clip(request.body, redactor) },
+      response: { status: answer.status, body: clip(body, redactor) },
+    });
+    return answer;
+  }
+
+  return { name: 'http', way, base, call, async close() {} };
+}

package/tests/drivers/index.mjs

@@ -0,0 +1,65 @@
+// Builds the driver for a way in from its entry in tests.yaml. A driver that
+// the kit does not ship (a browser, a command) comes from the adapter:
+//   adapter.drivers = { browser: async (ctx) => driver }
+// A driver has the shape { name, way, call(request, opts), close() }.
+import { isAbsolute, resolve } from 'node:path';
+import { Blocked } from '../errors.mjs';
+import { requireAddressesAllowed, substitute } from '../guards.mjs';
+import { createHttpDriver } from './http.mjs';
+import { createMcpStdioDriver } from './mcp-stdio.mjs';
+import { createFunctionDriver } from './function.mjs';
+
+export { createHttpDriver, createMcpStdioDriver, createFunctionDriver };
+
+// ctx: { way, spec, env, root, hosts, redactor, record, adapter, runId }
+// In a fault run, ATLAS_PRODUCT_ROOT names a throwaway copy. A process that a
+// way in starts (an MCP server) and the module of the function driver load from
+// that copy, so a planted fault is what runs. tests.yaml and the kit stay in the
+// real worktree. A `cwd` in the way in is a folder under that root.
+export function productRootOf(root) {
+  return process.env.ATLAS_PRODUCT_ROOT ? resolve(process.env.ATLAS_PRODUCT_ROOT) : root;
+}
+
+// Moves a path in a command from the worktree to the product root. The text
+// is changed only where it names the worktree root as a whole path part.
+export function commandFor(command, root, productRoot) {
+  if (!command || productRoot === root) return command;
+  return command.split(root).join(productRoot);
+}
+
+export async function makeDriver(ctx) {
+  const { way, spec, env, root, hosts, redactor, record, adapter = {}, runId } = ctx;
+  const productRoot = productRootOf(root);
+  const own = adapter.drivers?.[way] ?? adapter.drivers?.[spec.driver];
+  if (own) return own(ctx);
+  const template = { id: runId, varsFile: env.varsFile, url: env.url, vars: (n) => env.vars[n], standIn: (n) => env.standIns[n]?.url };
+  const fill = (text) => (text === undefined ? text : substitute(text, template));
+  // Every address in the env of a way in must be on the list, of any scheme and whatever its name.
+  if (spec.env && typeof spec.env === 'object') {
+    requireAddressesAllowed(Object.fromEntries(Object.entries(spec.env).map(([k, v]) => [k, fill(String(v))])), hosts, `the env of the way in "${way}"`);
+  }
+  switch (spec.driver) {
+    case 'http':
+      return createHttpDriver({ way, base: fill(spec.base), hosts, record, redactor });
+    case 'mcp-stdio':
+      return createMcpStdioDriver({
+        way, command: commandFor(fill(spec.command), root, productRoot), cwd: spec.cwd ? (isAbsolute(spec.cwd) ? spec.cwd : resolve(productRoot, spec.cwd)) : productRoot, hosts, redactor, record,
+        env: Object.fromEntries(Object.entries(spec.env ?? {}).map(([k, v]) => [k, fill(String(v))])),
+      });
+    case 'function': {
+      let target = adapter.functions?.[way] ?? adapter.functions?.function ?? adapter.functions;
+      // No module from the adapter: the `module` of the way in is a path under the
+      // product root. The adapter loads it (adapter.loadModule(file)), because the
+      // kit loads no module by a name it builds at run time.
+      if (!target && spec.module) {
+        const file = resolve(productRoot, fill(String(spec.module)));
+        if (typeof adapter.loadModule !== 'function') throw new Blocked(`the way in "${way}" names the module ${spec.module}; the adapter gives no loadModule(file) to load it from ${productRoot}`);
+        try { target = await adapter.loadModule(file, { way, root, productRoot }); } catch (error) { throw new Blocked(`the module of "${way}" could not be loaded from ${file}: ${error.message}`); }
+      }
+      if (!target) throw new Blocked(`the way in "${way}" uses the function driver and the adapter gives no module: set adapter.functions`);
+      return createFunctionDriver({ way, target, record, redactor });
+    }
+    default:
+      throw new Blocked(`the way in "${way}" uses the driver "${spec.driver}", which the kit does not ship; add it to adapter.drivers`);
+  }
+}

package/tests/drivers/mcp-stdio.mjs

@@ -0,0 +1,174 @@
+// The mcp-stdio driver: a small JSON-RPC 2.0 client for an MCP server that
+// speaks over standard input and output, one JSON message on each line (the
+// MCP stdio transport). Atlas has no dependency, so it does not use the
+// official client library. It does what the tests need: initialize, the
+// initialized notification, tools/list and tools/call.
+//
+// The seven duties of a driver (design section 7.2):
+//   1. Before it starts the server, it checks every address in the env that it
+//      gives the server against guards.hosts. A server that gets no address is
+//      refused, because it could fall back to a real one. The result is `blocked`.
+//   2. A credential goes only into the env of the server it belongs to. Each
+//      distinct env starts its own server process.
+//   3. It records each tool call and each answer.
+//   4. Recorded text has run secrets and named secrets removed, by value.
+//   5. It returns the raw answer: { status, body, isError, raw }.
+//      `body` is the result of tools/call, or the JSON-RPC error. `raw` is the whole message.
+//      Reading the product's error form is the work of the actions.
+//   6. It signs in as a persona: the env of the spec may hold {persona.credential}.
+//   7. It stops each server it started, on close and when the process ends.
+import { Blocked, Unavailable } from '../errors.mjs';
+import { childEnv, requireLocalAddresses } from '../guards.mjs';
+import { spawnManaged } from '../procs.mjs';
+import { clip } from '../evidence.mjs';
+
+const PROTOCOL = '2025-06-18';
+// The most that one line of the server may hold. A longer line is refused.
+export const MAX_LINE_BYTES = 4 * 1024 * 1024;
+
+class Session {
+  constructor({ proc, timeoutMs }) {
+    this.proc = proc;
+    this.timeoutMs = timeoutMs;
+    this.nextId = 1;
+    this.pending = new Map();
+    this.buffer = '';
+    this.closed = false;
+    proc.child.stdout.on('data', (chunk) => this.onData(chunk.toString('utf8')));
+    proc.child.stdin.on('error', () => { /* the server stopped; the exit handler reports it */ });
+    proc.exited.then(({ code, signal }) => {
+      this.closed = true;
+      for (const { reject } of this.pending.values()) reject(new Unavailable(`the MCP server stopped (${code ?? signal}). Last output: ${proc.tail().slice(-400)}`));
+      this.pending.clear();
+    });
+  }
+
+  overflow() {
+    this.buffer = '';
+    this.closed = true;
+    const why = new Unavailable(`the MCP server sent a line over ${MAX_LINE_BYTES / (1024 * 1024)} MB; the line was refused and the server was stopped`);
+    for (const { reject, timer } of this.pending.values()) { clearTimeout(timer); reject(why); }
+    this.pending.clear();
+    this.proc.stop();
+  }
+
+  onData(text) {
+    if (this.closed) return;
+    this.buffer += text;
+    let at;
+    while ((at = this.buffer.indexOf('\n')) >= 0) {
+      if (at > MAX_LINE_BYTES) return this.overflow();
+      const line = this.buffer.slice(0, at).trim();
+      this.buffer = this.buffer.slice(at + 1);
+      if (!line) continue;
+      let message;
+      try { message = JSON.parse(line); } catch { continue; }
+      this.onMessage(message);
+    }
+    if (this.buffer.length > MAX_LINE_BYTES) this.overflow();
+  }
+
+  onMessage(message) {
+    if (message.id !== undefined && (message.result !== undefined || message.error !== undefined)) {
+      const waiting = this.pending.get(message.id);
+      if (waiting) { this.pending.delete(message.id); clearTimeout(waiting.timer); waiting.resolve(message); }
+    } else if (message.id !== undefined && message.method) {
+      // The client offers nothing to a server that asks.
+      this.send({ jsonrpc: '2.0', id: message.id, error: { code: -32601, message: 'method not found' } });
+    }
+  }
+
+  send(message) {
+    if (this.closed) throw new Unavailable('the MCP server is not running');
+    this.proc.child.stdin.write(`${JSON.stringify(message)}\n`);
+  }
+
+  request(method, params) {
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new Unavailable(`the MCP server did not answer ${method} in ${this.timeoutMs} ms`));
+      }, this.timeoutMs);
+      this.pending.set(id, { resolve, reject, timer });
+      try { this.send({ jsonrpc: '2.0', id, method, params }); } catch (error) { clearTimeout(timer); this.pending.delete(id); reject(error); }
+    });
+  }
+
+  notify(method, params) { this.send({ jsonrpc: '2.0', method, params }); }
+
+  async open() {
+    // A server that stops or stays mute before it answers initialize never started: `blocked`.
+    let init;
+    try { init = await this.request('initialize', { protocolVersion: PROTOCOL, capabilities: {}, clientInfo: { name: 'atlas-tests', version: '1' } }); } catch (error) {
+      if (error instanceof Unavailable) throw new Blocked(`the MCP server did not start: ${error.message}`);
+      throw error;
+    }
+    if (init.error) throw new Blocked(`the MCP server refused initialize: ${init.error.message}`);
+    this.notify('notifications/initialized');
+    return init.result;
+  }
+
+  async close() {
+    this.closed = true;
+    try { this.proc.child.stdin.end(); } catch { /* it is closed */ }
+    await this.proc.stop();
+  }
+}
+
+export function createMcpStdioDriver({ way, command, env: specEnv = {}, cwd, hosts, redactor, record: rawRecord, timeoutMs = 30000 }) {
+  const record = rawRecord && ((call) => rawRecord(redactor ? redactor.deep(call) : call));
+  if (!command) throw new Error(`the way in "${way}" has no command`);
+  const sessions = new Map();
+
+  const fill = (credential) => Object.fromEntries(Object.entries(specEnv).map(([k, v]) => [k, String(v).replaceAll('{persona.credential}', credential ?? '')]));
+
+  async function sessionFor(credential) {
+    const given = fill(credential);
+    const key = JSON.stringify(given);
+    if (sessions.has(key)) return sessions.get(key);
+    // Duty 1: check the address before the server starts.
+    // The check runs on the whole env the server gets, not only on the part the way in sets.
+    const full = childEnv(process.env, given);
+    requireLocalAddresses(given, hosts, `the MCP server of "${way}"`, full);
+    const proc = spawnManaged({ command, cwd, redactor, stdin: true, env: full });
+    const session = new Session({ proc, timeoutMs });
+    sessions.set(key, session);
+    try { await session.open(); } catch (error) { sessions.delete(key); await session.close(); throw error; }
+    return session;
+  }
+
+  // request: { tool, arguments } for tools/call, or { method, params } for any other call.
+  async function call(request = {}, { credential = null, persona = null } = {}) {
+    const session = await sessionFor(credential);
+    const method = request.tool ? 'tools/call' : request.method;
+    const params = request.tool ? { name: request.tool, arguments: request.arguments ?? {} } : request.params;
+    if (!method) throw new Error('an MCP request names a tool, or a method');
+    const message = await session.request(method, params);
+    const failed = message.error !== undefined;
+    const body = failed ? message.error : message.result;
+    const answer = {
+      status: failed ? message.error.code : (body?.isError ? 'tool-error' : 'ok'),
+      body, isError: failed || body?.isError === true, raw: message,
+    };
+    record?.({
+      way, persona,
+      request: { method, params: clip(params, redactor) },
+      response: { status: answer.status, body: clip(body, redactor) },
+    });
+    return answer;
+  }
+
+  async function listTools(opts) {
+    const answer = await call({ method: 'tools/list', params: {} }, opts);
+    return answer.body?.tools ?? [];
+  }
+
+  async function close() {
+    const all = [...sessions.values()];
+    sessions.clear();
+    await Promise.all(all.map((s) => s.close()));
+  }
+
+  return { name: 'mcp-stdio', way, call, listTools, close };
+}

package/tests/environment.mjs

@@ -0,0 +1,227 @@
+// The run environment: start the stand-ins, then the product; make the run
+// secrets and the vars file new for each run; wait until the product is
+// ready; check that the product called each stand-in; stop everything.
+//
+//   const env = await startEnvironment({ root, tests, name: 'local', runId, redactor, adapter });
+//   env.url, env.vars, env.standIns.engram.url
+//   await env.stop();
+//
+// Every child process starts from an allow-list environment (guards.mjs,
+// childEnv), and each resolved var is set as a real env value on the product.
+// The vars file lives in a new temporary folder and is removed on stop.
+import { mkdtempSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { dirname, join, parse } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { spawnManaged } from './procs.mjs';
+import { Blocked } from './errors.mjs';
+import { resolveEnvironments } from './tests-yaml.mjs';
+import { childEnv, hostsOf, newSecret, requireAddressesAllowed, requireAllowed, substitute } from './guards.mjs';
+import { CALLS_PATH, countCalls, LIBRARY_HEADER } from './stand-in.mjs';
+import { parseLimit } from './wait.mjs';
+
+const LIBRARY_HEADERS = { [LIBRARY_HEADER]: '1' };
+
+function freePort() {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => { const { port } = server.address(); server.close(() => resolve(port)); });
+  });
+}
+
+// ---------------------------------------------------------------- ready
+
+async function pollReady(spec, ctx, { label, process: proc, hosts, redactor, standInOrigins = [] }) {
+  const url = substitute(spec.get ?? '', ctx);
+  if (!url) throw new Blocked(`${label}: "ready" has no "get" address`);
+  const parsed = requireAllowed(url, hosts, `${label} ready check`);
+  // A ready address must not point at a stand-in: a poll there could stand in
+  // for the first call of the product.
+  if (standInOrigins.includes(parsed.origin)) throw new Blocked(`${label}: the "ready" address ${parsed.origin} is a stand-in; the ready check must ask the product itself`);
+  const want = spec.status ?? 200;
+  const limit = parseLimit(spec.limit ?? '30s');
+  const deadline = Date.now() + limit;
+  let last = 'no answer';
+  for (;;) {
+    if (proc && (proc.child.exitCode !== null || proc.child.signalCode !== null)) {
+      throw new Blocked(`${label} stopped before it was ready (${proc.child.exitCode ?? proc.child.signalCode}). Last output: ${proc.tail().slice(-600)}`);
+    }
+    try {
+      const res = await fetch(url, { redirect: 'manual', headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(2000) });
+      await res.arrayBuffer();
+      if (res.status === want) return;
+      last = `status ${res.status}`;
+    } catch (error) {
+      last = error.cause?.code ?? error.name;
+    }
+    if (Date.now() >= deadline) {
+      throw new Blocked(`${label} was not ready after ${spec.limit ?? '30s'} (${last}). Last output: ${redactor?.text(proc?.tail() ?? '').slice(-600)}`);
+    }
+    await sleep(150);
+  }
+}
+
+// ---------------------------------------------------------------- secrets
+
+// KEY=value lines of a dotenv file. No expansion.
+export function parseDotenv(text) {
+  const out = {};
+  for (const line of String(text).split('\n')) {
+    const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/.exec(line);
+    if (!m || line.trim().startsWith('#')) continue;
+    let value = m[2];
+    if (/^(["']).*\1$/.test(value)) value = value.slice(1, -1);
+    out[m[1]] = value;
+  }
+  return out;
+}
+
+function findDotenv(from) {
+  let dir = from;
+  for (;;) {
+    const file = join(dir, '.env.test');
+    if (existsSync(file)) return file;
+    if (existsSync(join(dir, '.git')) || dir === parse(dir).root) return null;
+    dir = dirname(dir);
+  }
+}
+
+// The value of each provided secret that the scenario names. On the Mac it is
+// in .env.test at the repo root. In CI it is a CI secret of the same name.
+// With no value, the result is `blocked` and names the key.
+export function provideSecrets(names, { root, tests, env = process.env }) {
+  const out = {};
+  if (!names.length) return out;
+  const file = findDotenv(root);
+  const fromFile = file ? parseDotenv(readFileSync(file, 'utf8')) : {};
+  for (const name of names) {
+    if (!Object.hasOwn(tests.secrets ?? {}, name)) continue;
+    const value = fromFile[name] ?? env[name];
+    if (!value) throw new Blocked(`the scenario names the key ${name} and no value exists: add it to .env.test at the repo root, or set the CI secret of the same name`);
+    out[name] = value;
+  }
+  return out;
+}
+
+const dotenvLine = (name, value) => `${name}=${/^[A-Za-z0-9_./:@%+=,-]*$/.test(value) ? value : JSON.stringify(value)}\n`;
+
+// ---------------------------------------------------------------- start
+
+export async function startEnvironment({ root, tests, name, runId, redactor, adapter = {}, provided = {} }) {
+  const environments = resolveEnvironments(tests.environments ?? {});
+  const spec = environments[name];
+  if (!spec) throw new Blocked(`tests.yaml has no environment "${name}"`);
+  if (spec.allowed === false) throw new Blocked(`the environment "${name}" is not allowed: "allowed" is false until a test environment and a scoped key exist`);
+  // A deployed environment is closed until the owner opens it with `allowed: true`.
+  if (name === 'deployed' && spec.allowed !== true) throw new Blocked(`the environment "deployed" is closed: it has no "allowed: true". Only the owner opens it, with a test environment and a scoped key`);
+  const hosts = hostsOf(tests);
+  const workDir = mkdtempSync(join(tmpdir(), 'atlas-run-'));
+  const varsFile = join(workDir, 'vars.env');
+  const managed = [];
+  const stoppers = [];
+  const standIns = {};
+  const resolvedVars = {};
+  const secretsMade = [];
+  for (const value of Object.values(provided)) redactor.add(value);
+
+  const stop = async () => {
+    for (const fn of stoppers.reverse()) { try { await fn(); } catch { /* the stop goes on */ } }
+    for (const proc of managed.reverse()) await proc.stop(proc.stopCommand);
+    rmSync(workDir, { recursive: true, force: true });
+  };
+
+  try {
+    // 1. The stand-ins.
+    for (const [standInName, def] of Object.entries(tests['stand-ins'] ?? {})) {
+      const label = `stand-in "${standInName}"`;
+      if (adapter.standIns?.[standInName]) {
+        const made = await adapter.standIns[standInName]({ name: standInName, runId, hosts });
+        if (made.stop) stoppers.push(made.stop);
+        requireAllowed(made.url, hosts, label);
+        standIns[standInName] = { url: made.url, calls: made.calls ?? (async () => countCalls(await (await fetch(`${made.url}${CALLS_PATH}`, { headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(3000) })).json())) };
+        continue;
+      }
+      if (!def?.start) throw new Blocked(`${label} has no "start" command and the adapter does not make it`);
+      const port = await freePort();
+      const url = `http://127.0.0.1:${port}`;
+      const ctx = { id: runId, port, url };
+      const proc = spawnManaged({
+        command: substitute(def.start, ctx), cwd: root, redactor,
+        env: childEnv(process.env, { PORT: port, ATLAS_STAND_IN_PORT: port, ATLAS_RUN_ID: runId }),
+      });
+      managed.push(proc);
+      await pollReady(def.ready ?? { get: `${url}${CALLS_PATH}`, status: 200, limit: '30s' }, ctx, { label, process: proc, hosts, redactor, standInOrigins: Object.values(standIns).map((one) => new URL(one.url).origin) });
+      standIns[standInName] = {
+        url,
+        calls: async () => countCalls(await (await fetch(`${url}${CALLS_PATH}`, { headers: LIBRARY_HEADERS, signal: AbortSignal.timeout(3000) })).json()),
+      };
+    }
+
+    // 2. The vars: run secrets are new for each placeholder; references resolve in order.
+    const raw = spec.vars ?? {};
+    // The product gets a free port too: `{port}` in start, ready, url and vars.
+    const productPort = await freePort();
+    const ctx = {
+      id: runId, varsFile, port: productPort, url: spec.url ? substitute(spec.url, { id: runId, port: productPort }) : undefined,
+      secret: () => { const s = newSecret(); secretsMade.push(s); redactor.add(s); return s; },
+      standIn: (n) => standIns[n]?.url,
+      vars: (n) => resolveVar(n, []),
+    };
+    function resolveVar(varName, stack) {
+      if (Object.hasOwn(resolvedVars, varName)) return resolvedVars[varName];
+      if (!Object.hasOwn(raw, varName)) throw new Blocked(`a placeholder names the variable ${varName}, which "vars" does not set`);
+      if (stack.includes(varName)) throw new Blocked(`the variables ${[...stack, varName].join(' -> ')} refer to each other in a circle`);
+      const value = substitute(String(raw[varName]), { ...ctx, vars: (n) => resolveVar(n, [...stack, varName]) });
+      resolvedVars[varName] = value;
+      return value;
+    }
+    for (const varName of Object.keys(raw)) resolveVar(varName, []);
+    for (const [varName, value] of Object.entries(resolvedVars)) {
+      if (/(TOKEN|SECRET|KEY|PASSWORD|CREDENTIAL)/i.test(varName)) redactor.add(value);
+    }
+    // Every value that is an address, of any scheme or as host:port, must be on
+    // the list, whatever the name of its variable.
+    requireAddressesAllowed(resolvedVars, hosts, 'the vars of the environment');
+    writeFileSync(varsFile, Object.entries(resolvedVars).map(([n, v]) => dotenvLine(n, v)).join(''), { mode: 0o600 });
+
+    // 3. The product. In a fault run, ATLAS_PRODUCT_ROOT names the folder of a
+    // throwaway copy: the product starts there, while tests.yaml, the support
+    // code and the stand-ins stay in the real worktree.
+    const productRoot = process.env.ATLAS_PRODUCT_ROOT || root;
+    const given = { ...provided, ATLAS_RUN_ID: runId };
+    let product = null;
+    if (spec.start) {
+      // An allow-list environment. Each resolved var is a real value that
+      // replaces any parent value of the same name; the vars file stays too.
+      product = spawnManaged({
+        command: substitute(spec.start, ctx), cwd: productRoot, redactor,
+        env: childEnv(process.env, { ...resolvedVars, ...given }),
+      });
+      product.stopCommand = spec.stop;
+      managed.push(product);
+    }
+    if (spec.ready) await pollReady(spec.ready, ctx, { label: `the product (${name})`, process: product, hosts, redactor, standInOrigins: Object.values(standIns).map((one) => new URL(one.url).origin) });
+    if (spec.url) requireAllowed(substitute(spec.url, ctx), hosts, 'the environment url');
+
+    const env = {
+      name, url: spec.url ? substitute(spec.url, ctx) : null, port: productPort, vars: { ...resolvedVars }, varsFile, hosts, standIns, root, productRoot,
+      provided: Object.keys(provided), output: () => product?.tail() ?? '', stop,
+    };
+
+    // 4. The first-call rule: the product must have called each stand-in.
+    await adapter.warmUp?.(env);
+    for (const [standInName, one] of Object.entries(standIns)) {
+      let count = 0;
+      try { count = await one.calls(); } catch (error) {
+        throw new Blocked(`stand-in "${standInName}" does not answer ${CALLS_PATH}: ${error.cause?.code ?? error.message}`);
+      }
+      if (count < 1) throw new Blocked(`stand-in "${standInName}" saw no first call from the product before step 1: the product may talk to something else`);
+    }
+    return env;
+  } catch (error) {
+    await stop();
+    throw error;
+  }
+}

package/tests/errors.mjs

@@ -0,0 +1,27 @@
+// The kinds of "could not run" and the other stops. Each one has one result.
+
+// A setup failed, a guard refused, or a precondition does not hold. The
+// result is `blocked`, never `fail`.
+export class Blocked extends Error {
+  constructor(reason) { super(reason); this.name = 'Blocked'; }
+}
+
+// A service that the test needs does not answer. The result is `blocked`.
+export class Unavailable extends Error {
+  constructor(reason) { super(reason); this.name = 'Unavailable'; }
+}
+
+// A wait met its fail state. This is a product result, so it is a `fail`.
+export class WaitFailed extends Error {
+  constructor(reason, last) { super(reason); this.name = 'WaitFailed'; this.last = last; }
+}
+
+// A wait ran out of time. This is a product result, so it is a `fail`.
+export class WaitTimeout extends Error {
+  constructor(reason, last) { super(reason); this.name = 'WaitTimeout'; this.last = last; }
+}
+
+// Thrown by a failed gate. It stops the body of the test.
+export class GateFailed extends Error {
+  constructor(reason) { super(reason); this.name = 'GateFailed'; }
+}