@arjunkhera/atlas 0.3.6 → 0.3.8

package/tests/evidence.mjs

@@ -0,0 +1,113 @@
+// The evidence file. Schema `evidence: 1`.
+//
+//   {
+//     "evidence": 1, "run_id": "...", "environment": "local",
+//     "scenario": "week-conflict", "title": "...", "source": "scenarios/week-conflict.md",
+//     "source_sha256": "...", "fingerprint_method": "fp1",
+//     "started": "...", "ended": null, "verdict": "not finished",
+//     "ways": { "mcp": { "verdict": "pass", "reason": null, "started": "...", "ended": "..." } },
+//     "assertions": [ { "id": "week-conflict/e3#09f598", "way": "mcp", "result": "pass", "proof": { ... } } ],
+//     "tables": { "T1": "<fingerprint>" }, "fresh": [ { "rule": "...", "value": "..." } ],
+//     "calls": [ { "n": 1, "way": "mcp", "persona": "planner-1", "request": { }, "response": { } } ],
+//     "notes": [], "files": [], "provided_secrets": [], "fault_run": null
+//   }
+//
+// The file is written when the run starts and after each assertion, so a run
+// that dies leaves a record with the verdict "not finished". One file for each
+// run and scenario. A second attempt in the same run id gets a new file
+// (`<scenario>.2.json`). A file is claimed with an exclusive create, so no
+// earlier run is overwritten. Every string is redacted by value before it is
+// written.
+import { closeSync, mkdirSync, openSync, renameSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+export const EVIDENCE_SCHEMA = 1;
+
+// The results of one assertion for one way in (design section 6.11).
+export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here']);
+
+const MAX_BODY = 8000;
+
+// Redact first, then cut. A cut before the redaction could split a secret, and
+// the half that stays would no longer match its value.
+export function clip(value, redactor = null) {
+  const safe = redactor ? redactor.deep(value) : value;
+  const text = typeof safe === 'string' ? safe : JSON.stringify(safe);
+  if (text === undefined || text.length <= MAX_BODY) return safe;
+  return `${text.slice(0, MAX_BODY)}... [${text.length - MAX_BODY} more characters]`;
+}
+
+export class Evidence {
+  constructor({ evidenceDir, runId, environment, scenario, root, redactor, providedSecrets = [] }) {
+    this.dir = join(evidenceDir, runId);
+    this.root = root;
+    this.redactor = redactor;
+    this.record = {
+      evidence: EVIDENCE_SCHEMA, run_id: runId, environment,
+      scenario: scenario.id, title: scenario.title,
+      source: relative(root, scenario.file).split('\\').join('/'), source_sha256: scenario.sha256,
+      fingerprint_method: scenario.method,
+      started: new Date().toISOString(), ended: null, verdict: 'not finished',
+      ways: {}, assertions: [], tables: {}, fresh: [], calls: [], notes: [], files: [],
+      provided_secrets: providedSecrets, fault_run: process.env.ATLAS_FAULT_RUN || null,
+    };
+    for (const way of scenario.through) {
+      this.record.ways[way] = { verdict: 'not finished', reason: null, started: null, ended: null };
+      for (const a of scenario.assertions) this.record.assertions.push({ id: a.fullId, way, result: 'not checked', proof: null });
+    }
+    mkdirSync(this.dir, { recursive: true });
+    this.path = this.claim(scenario.id);
+    this.write();
+  }
+
+  claim(scenarioId) {
+    for (let n = 1; ; n += 1) {
+      const path = join(this.dir, n === 1 ? `${scenarioId}.json` : `${scenarioId}.${n}.json`);
+      try { closeSync(openSync(path, 'wx')); return path; } catch (error) { if (error.code !== 'EEXIST') throw error; }
+    }
+  }
+
+  entry(id, way) {
+    const found = this.record.assertions.find((a) => a.id === id && a.way === way);
+    if (!found) throw new Error(`the evidence has no assertion ${id} for the way in "${way}"`);
+    return found;
+  }
+
+  setResult(id, way, result, proof) {
+    if (!RESULTS.includes(result)) throw new Error(`"${result}" is not a result`);
+    const entry = this.entry(id, way);
+    entry.result = result;
+    entry.proof = proof ?? null;
+    this.write();
+  }
+
+  setWay(way, patch) {
+    Object.assign(this.record.ways[way], patch);
+    const verdicts = Object.values(this.record.ways).map((w) => w.verdict);
+    this.record.verdict = verdicts.includes('fail') ? 'fail'
+      : verdicts.includes('blocked') ? 'blocked'
+        : verdicts.every((v) => v === 'pass' || v === 'not here') ? (verdicts.every((v) => v === 'not here') ? 'not here' : 'pass') : 'not finished';
+    if (!verdicts.includes('not finished')) this.record.ended = new Date().toISOString();
+    this.write();
+  }
+
+  addCall(call) {
+    const n = this.record.calls.length + 1;
+    this.record.calls.push({ n, ...call });
+    return n;
+  }
+
+  addFile(path) {
+    this.record.files.push(relative(this.dir, path).split('\\').join('/'));
+    this.write();
+  }
+
+  note(text) { this.record.notes.push(String(text)); this.write(); }
+
+  write() {
+    const safe = this.redactor.deep(this.record);
+    const temp = `${this.path}.tmp`;
+    writeFileSync(temp, `${JSON.stringify(safe, null, 2)}\n`);
+    renameSync(temp, this.path);
+  }
+}

package/tests/fresh.mjs

@@ -0,0 +1,42 @@
+// Fresh values. Every value that a test makes carries the run id, so two runs
+// never share data, and a clean-up can find what a killed run left behind.
+// A value that cannot hold text, such as a date, is picked from the run id.
+import { createHash, randomBytes } from 'node:crypto';
+
+export function makeRunId(env = process.env) {
+  if (env.ATLAS_RUN_ID) return env.ATLAS_RUN_ID;
+  if (env.GITHUB_RUN_ID) return `ci${env.GITHUB_RUN_ID}-${env.GITHUB_RUN_ATTEMPT ?? 1}`;
+  return `run${randomBytes(3).toString('hex')}`;
+}
+
+// The first Monday after 1 January 2035, then a number of whole weeks.
+const FIRST_MONDAY = Date.UTC(2035, 0, 1) + (((8 - new Date(Date.UTC(2035, 0, 1)).getUTCDay()) % 7) || 7) * 86400000;
+const SPREAD_WEEKS = 520;
+
+export function mondayFor(runId, count) {
+  const weeks = createHash('sha256').update(`${runId}:${count}`).digest().readUInt32BE(0) % SPREAD_WEEKS;
+  return new Date(FIRST_MONDAY + weeks * 7 * 86400000).toISOString().slice(0, 10);
+}
+
+// A maker of fresh values for one run. `rules` is the `data` map of tests.yaml.
+// A rule whose name holds "monday" makes a Monday. Any other rule makes a
+// name: the rule, the run id and a count.
+export function makeFresh(runId, rules, adapterRules = {}) {
+  const counts = new Map();
+  const made = [];
+  const fresh = (rule) => {
+    if (!Object.hasOwn(rules ?? {}, rule) && !Object.hasOwn(adapterRules, rule)) {
+      throw new Error(`there is no data rule "${rule}" in tests.yaml`);
+    }
+    const count = (counts.get(rule) ?? 0) + 1;
+    counts.set(rule, count);
+    let value;
+    if (Object.hasOwn(adapterRules, rule)) value = adapterRules[rule]({ runId, count, rule });
+    else if (/monday/i.test(rule)) value = mondayFor(runId, `${rule}:${count}`);
+    else value = `${rule.replace(/^fresh-/, '')}-${runId}-${count}`;
+    made.push({ rule, value });
+    return value;
+  };
+  fresh.made = made;
+  return fresh;
+}

package/tests/guards.mjs

@@ -0,0 +1,159 @@
+// Guards: what a test may reach, and what a child process may see.
+import { Blocked } from './errors.mjs';
+import { randomBytes } from 'node:crypto';
+
+export const LOCAL_HOSTS = Object.freeze(['127.0.0.1', 'localhost', '::1']);
+
+const bare = (host) => String(host).toLowerCase().replace(/^\[|\]$/g, '');
+
+// The hosts that a test may reach: `guards.hosts`, or this machine only.
+export function hostsOf(tests) {
+  const hosts = tests?.guards?.hosts;
+  return Array.isArray(hosts) && hosts.length ? hosts.map(String) : [...LOCAL_HOSTS];
+}
+
+export function hostAllowed(host, hosts) {
+  const h = bare(host);
+  return hosts.some((entry) => {
+    const e = bare(entry);
+    return e === h || (e.startsWith('*.') && h.endsWith(e.slice(1)));
+  });
+}
+
+// Throws Blocked when the address is not on the list. The result is `blocked`,
+// never `fail`.
+export function requireAllowed(url, hosts, what = 'a request') {
+  let parsed;
+  try { parsed = new URL(url); } catch { throw new Blocked(`${what}: "${String(url).slice(0, 80)}" is not an address`); }
+  if (!hostAllowed(parsed.hostname, hosts)) {
+    throw new Blocked(`${what} to ${parsed.origin} was refused: guards.hosts allows only ${hosts.join(', ')}`);
+  }
+  return parsed;
+}
+
+export const sameOrigin = (a, b) => new URL(a).origin === new URL(b).origin;
+
+// ---------------------------------------------------------------- child env
+
+// A child process starts from an ALLOW-LIST, not from the parent environment
+// minus a few names (design section 15, rule 1). It gets these names from the
+// parent and nothing else: PATH, HOME, LANG, LC_*, TMPDIR, TERM and NODE_*
+// (such as NODE_EXTRA_CA_CERTS). The rule follows packages/atlas/work/lib/child-env.mjs.
+// The proxy variables pass only in a Claude cloud session, and only when the
+// proxy is a loopback address. NODE_TEST_CONTEXT never passes.
+const ALLOWED = /^(?:PATH|HOME|LANG|LC_[A-Z_]+|TMPDIR|TERM)$/;
+const CLOUD_PROXY = ['HTTPS_PROXY', 'https_proxy'];
+const CLOUD_NO_PROXY = ['NO_PROXY', 'no_proxy'];
+const LOOPBACK_PROXY = /^http:\/\/(127\.0\.0\.1|localhost|\[::1\]):\d{1,5}\/?$/;
+
+// The environment of a child process: the allow-list from `parent`, then
+// `values` on top. A value in `values` replaces any parent value of the same
+// name, so a parent value for a name that tests.yaml sets never reaches a child.
+export function childEnv(parent = process.env, values = {}) {
+  const out = {};
+  for (const [name, value] of Object.entries(parent)) {
+    if (value === undefined) continue;
+    if (ALLOWED.test(name) || (name.startsWith('NODE_') && name !== 'NODE_TEST_CONTEXT')) out[name] = value;
+  }
+  if (parent.CLAUDE_CODE_REMOTE === 'true') {
+    const proxies = CLOUD_PROXY.filter((name) => parent[name] !== undefined);
+    if (proxies.length && proxies.every((name) => LOOPBACK_PROXY.test(parent[name]))) {
+      for (const name of [...proxies, ...CLOUD_NO_PROXY]) if (parent[name] !== undefined) out[name] = parent[name];
+    }
+  }
+  for (const [name, value] of Object.entries(values)) if (value !== undefined) out[name] = String(value);
+  return out;
+}
+
+// ---------------------------------------------------------------- addresses
+
+const ADDRESS_NAME = /(?:URL|URI|HOST|ADDR|ADDRESS|ENDPOINT)/i;
+const SCHEME_URL = /^[a-z][a-z0-9+.-]*:\/\/\S+/i;
+const HOST_PORT = /^(\[[0-9a-f:.]+\]|[a-z0-9][a-z0-9.-]*):(\d{1,5})$/i;
+
+// The host of one token, when the token is an address: a URL of any scheme
+// (a file URL has no host and is no address), or host:port. Returns
+// undefined when the token is not an address, and null when it is a URL that
+// does not parse.
+function hostOfToken(token) {
+  if (SCHEME_URL.test(token)) {
+    try { return new URL(token).hostname || undefined; } catch { return null; }
+  }
+  const m = HOST_PORT.exec(token);
+  if (!m) return undefined;
+  const host = m[1];
+  // "12:30" is a time, not an address: a host needs a letter, a dot or brackets.
+  if (!/[a-z.\[]/i.test(host)) return undefined;
+  return host;
+}
+
+// The addresses in an env map. A value is an address when it parses as a URL
+// of any scheme or as host:port, whatever its name. A value may hold a list
+// (split on commas, semicolons and white space). With `byName`, the value of
+// a variable whose name says it is an address (URL, HOST, ...) is one too,
+// even when it parses as nothing.
+export function addressesIn(env = {}, { byName = true } = {}) {
+  const out = [];
+  for (const [key, value] of Object.entries(env)) {
+    const text = String(value ?? '').trim();
+    let found = false;
+    for (const token of text.split(/[\s,;]+/).filter(Boolean)) {
+      const host = hostOfToken(token);
+      if (host === undefined) continue;
+      found = true;
+      out.push({ key, value: token, host });
+    }
+    if (!found && byName && ADDRESS_NAME.test(key)) {
+      out.push({ key, value: text, host: text.replace(/:\d+$/, '').trim() || null });
+    }
+  }
+  return out;
+}
+
+// Every address in `env` must be on the list (`blocked` otherwise). Used for
+// the vars of an environment and for the env of a way in.
+export function requireAddressesAllowed(env, hosts, what) {
+  for (const one of addressesIn(env, { byName: false })) {
+    if (!one.host) throw new Blocked(`${what}: ${one.key} holds an address that cannot be read`);
+    if (!hostAllowed(one.host, hosts)) throw new Blocked(`${what}: ${one.key} points at ${one.host}, which guards.hosts does not allow (it allows only ${hosts.join(', ')})`);
+  }
+}
+
+// For a process that the library starts for a test, such as an MCP server:
+// every address in the env that the library gives it must be on the list, and
+// there must be one. A server with no address could fall back to a real one.
+// `given` is what the way in sets. `full` is the whole env the server gets;
+// the list check runs on all of it.
+export function requireLocalAddresses(given, hosts, what, full = given) {
+  const own = addressesIn(given);
+  if (!own.length) throw new Blocked(`${what}: the env gives the server no address, so it could fall back to a real one; give it the local address`);
+  for (const one of addressesIn(full)) {
+    if (!one.host) throw new Blocked(`${what}: ${one.key} holds no address, so the server could fall back to a real one`);
+    if (!hostAllowed(one.host, hosts)) throw new Blocked(`${what}: ${one.key} points at ${one.host}, which guards.hosts does not allow`);
+  }
+}
+
+// ---------------------------------------------------------------- templates
+
+// Replaces the placeholders of tests.yaml. Known forms:
+//   {run.secret}  {run.id}  {run.vars}  {run.vars.NAME}  {stand-in.NAME.url}  {url}  {port}
+// `ctx` gives a function for each: secret(), id, varsFile, vars(name), standIn(name), url, port.
+const PLACEHOLDER = /\{(run\.secret|run\.id|run\.vars(?:\.[A-Za-z0-9_]+)?|stand-in\.[A-Za-z0-9_-]+\.url|url|port)\}/g;
+
+export function substitute(text, ctx) {
+  return String(text).replace(PLACEHOLDER, (whole, key) => {
+    let value;
+    if (key === 'run.secret') value = ctx.secret?.();
+    else if (key === 'run.id') value = ctx.id;
+    else if (key === 'run.vars') value = ctx.varsFile;
+    else if (key.startsWith('run.vars.')) value = ctx.vars?.(key.slice(9));
+    else if (key.startsWith('stand-in.')) value = ctx.standIn?.(key.slice(9, -4));
+    else if (key === 'url') value = ctx.url;
+    else if (key === 'port') value = ctx.port;
+    if (value === undefined || value === null || value === '') throw new Error(`no value for the placeholder ${whole}`);
+    return String(value);
+  });
+}
+
+// A fresh run secret: random, 40 characters.
+export const newSecret = () => randomBytes(30).toString('base64url').slice(0, 40);

package/tests/index.mjs

@@ -0,0 +1,11 @@
+// The public face of the kit. A repo imports from here, or from the files directly.
+export { scenario, defineAdapter, waitUntil, RUN_ID } from './contract.mjs';
+export { startEnvironment, parseDotenv, provideSecrets } from './environment.mjs';
+export { createStandIn, serveStandIn, CALLS_PATH, LIBRARY_HEADER } from './stand-in.mjs';
+export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, makeDriver } from './drivers/index.mjs';
+export { parseTestsYaml, readTestsYaml, checkTestsYaml, guardParts, compareGuardParts, GUARD_PARTS, looksLikeSecret } from './tests-yaml.mjs';
+export { parseScenario, readScenario, addFingerprints, FINGERPRINT_METHOD } from './scenario.mjs';
+export { Blocked, Unavailable, WaitFailed, WaitTimeout, GateFailed } from './errors.mjs';
+export { Redactor } from './redact.mjs';
+export { Evidence, EVIDENCE_SCHEMA, RESULTS } from './evidence.mjs';
+export { checkArea } from './link-check.mjs';