@arjunkhera/atlas 0.3.6 → 0.3.8

package/tests/redact.mjs

@@ -0,0 +1,58 @@
+// Redaction by value. A run holds a set of secret values: the run secrets,
+// the values of the named secrets, and every credential a persona holds.
+// Everything the library writes as evidence passes through it, so no value in
+// the set reaches an evidence file, a CI log or a pull request.
+//
+// Besides the set, it removes three shapes that are never evidence: a JSON
+// web token, a bearer credential and a signed query string.
+
+const SHAPES = [
+  [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, '[redacted:jwt]'],
+  [/\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 [redacted]'],
+  [/([?&](?:X-Amz-Signature|X-Amz-Credential|X-Amz-Security-Token|Signature|sig|access_token)=)[^&\s"']+/gi, '$1[redacted]'],
+];
+
+const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+export const MIN_SECRET_LENGTH = 4;
+
+export class Redactor {
+  constructor(values = []) {
+    this.values = new Set();
+    this.add(values);
+  }
+
+  // Adds a value or a list of values. A value shorter than MIN_SECRET_LENGTH
+  // is not added: it would damage ordinary text.
+  add(value) {
+    for (const one of [].concat(value ?? [])) {
+      if (typeof one === 'string' && one.length >= MIN_SECRET_LENGTH) this.values.add(one);
+    }
+    this.pattern = null;
+    return this;
+  }
+
+  text(input) {
+    if (typeof input !== 'string') return input;
+    if (!this.pattern) {
+      const sorted = [...this.values].sort((a, b) => b.length - a.length);
+      this.pattern = sorted.length ? new RegExp(sorted.map(escape).join('|'), 'g') : false;
+    }
+    let out = this.pattern ? input.replace(this.pattern, '[redacted]') : input;
+    for (const [shape, replacement] of SHAPES) out = out.replace(shape, replacement);
+    return out;
+  }
+
+  deep(value) {
+    if (typeof value === 'string') return this.text(value);
+    if (Array.isArray(value)) return value.map((one) => this.deep(one));
+    if (value instanceof Error) return this.text(value.message);
+    if (value && typeof value === 'object') {
+      return Object.fromEntries(Object.entries(value).map(([key, one]) => [this.text(key), this.deep(one)]));
+    }
+    return value;
+  }
+}
+
+// A name that says its value is a secret.
+export const SECRET_NAME = /(TOKEN|SECRET|KEY|PASSWORD|CREDENTIAL)/i;

package/tests/scenario.mjs

@@ -0,0 +1,325 @@
+// The scenario reader. A scenario is a Markdown file. Its front matter holds
+// the fixed parts. Its sections hold the words. Code reads these parts:
+// the front matter, the `Start with` items, the case tables, the numbered
+// steps (with their waits), and the assertion lines. Code never reads the
+// other words; agents and people do.
+//
+// Names in backticks are checked against tests.yaml by the link check.
+//
+// Fingerprints, method fp1 (design section 8.2, open question 17): the first
+// six hex characters of the SHA-256 of
+//   - the normalized words of the assertion, then
+//   - one line for each setup, fixture, data rule, `Start with` item and step
+//     that the assertion names, directly or through a step it names, then
+//   - the `through` and the `runs-in` lists of the front matter (the lead's
+//     answer to OQ-17), sorted. A change to either shows as a changed assertion.
+// Normalized means: NFC, LF line ends, every run of white space cut to one
+// space, trimmed. The rows of a case table are not in it. Each table has its
+// own fingerprint, so a new row adds a case with no code change.
+import { readFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { parseYaml, YamlError } from './yaml.mjs';
+import { parseLimit } from './wait.mjs';
+
+export const FINGERPRINT_METHOD = 'fp1';
+
+export const normalize = (text) => String(text).normalize('NFC').replace(/\r\n?/g, '\n').replace(/\s+/g, ' ').trim();
+export const sha256 = (text) => createHash('sha256').update(text).digest('hex');
+export const fingerprintOf = (text) => sha256(normalize(text)).slice(0, 6);
+
+const ASSERTION = /^- (e\d+) (exact|judged)(?:,\s*(gate))?:\s*(.+)$/;
+const TABLE_HEADING = /^###\s+(T\d+)\s*:?\s*(.*)$/;
+const WAIT_LIMIT = /give up after\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|seconds?|m|min|minutes?|h|hours?)\b/i;
+const names = (text) => [...String(text).matchAll(/`([^`]+)`/g)].map((m) => m[1]);
+
+const sortKeys = (x) => {
+  if (Array.isArray(x)) return x.map(sortKeys);
+  if (x && typeof x === 'object') return Object.fromEntries(Object.keys(x).sort().map((k) => [k, sortKeys(x[k])]));
+  return x;
+};
+
+// ---------------------------------------------------------------- tables
+
+function tableCells(line) {
+  return line.trim().replace(/^\||\|$/g, '').split('|').map((c) => c.trim());
+}
+
+// Every combination of the values, or a small set that holds every pair of values.
+export function combinations(dimensions) {
+  let out = [{}];
+  for (const [name, values] of Object.entries(dimensions)) out = out.flatMap((one) => values.map((value) => ({ ...one, [name]: value })));
+  return out;
+}
+
+export function pairs(dimensions) {
+  const names2 = Object.keys(dimensions);
+  if (names2.length < 2) return combinations(dimensions);
+  const key = (a, va, b, vb) => `${a}\u0000${va}\u0000${b}\u0000${vb}`;
+  const uncovered = new Set();
+  for (let i = 0; i < names2.length; i += 1) {
+    for (let j = i + 1; j < names2.length; j += 1) {
+      for (const va of dimensions[names2[i]]) for (const vb of dimensions[names2[j]]) uncovered.add(key(names2[i], va, names2[j], vb));
+    }
+  }
+  const gain = (partial, dim, value) => {
+    let n = 0;
+    for (const other of Object.keys(partial)) {
+      const [a, b] = names2.indexOf(other) < names2.indexOf(dim) ? [other, dim] : [dim, other];
+      const va = a === other ? partial[other] : value;
+      const vb = b === other ? partial[other] : value;
+      if (uncovered.has(key(a, va, b, vb))) n += 1;
+    }
+    return n;
+  };
+  const cases = [];
+  while (uncovered.size) {
+    const [seed] = uncovered;
+    const [a, va, b, vb] = seed.split('\u0000');
+    const one = { [a]: va, [b]: vb };
+    for (const dim of names2) {
+      if (dim in one) continue;
+      let best = dimensions[dim][0];
+      let bestGain = -1;
+      for (const value of dimensions[dim]) {
+        const g = gain(one, dim, value);
+        if (g > bestGain) { best = value; bestGain = g; }
+      }
+      one[dim] = best;
+    }
+    const ordered = Object.fromEntries(names2.map((n) => [n, one[n]]));
+    for (let i = 0; i < names2.length; i += 1) {
+      for (let j = i + 1; j < names2.length; j += 1) uncovered.delete(key(names2[i], ordered[names2[i]], names2[j], ordered[names2[j]]));
+    }
+    cases.push(ordered);
+  }
+  return cases;
+}
+
+function readTable(name, title, line, blockLines) {
+  const text = blockLines.map((l) => l.text);
+  const grids = [];
+  let current = null;
+  for (const { text: raw } of blockLines) {
+    if (raw.trim().startsWith('|')) {
+      const cells = tableCells(raw);
+      if (cells.every((c) => /^:?-+:?$/.test(c))) continue;
+      if (!current) { current = []; grids.push(current); }
+      current.push(cells);
+    } else current = null;
+  }
+  const dimensions = {};
+  let inDimensions = false;
+  let cover = null;
+  for (const { text: raw } of blockLines) {
+    const t = raw.trim();
+    if (/^Dimensions:/i.test(t)) { inDimensions = true; continue; }
+    if (/^Cover:/i.test(t)) { inDimensions = false; cover = /every pair/i.test(t) ? 'pair' : 'combination'; continue; }
+    if (inDimensions) {
+      const m = /^-\s*([^:]+):\s*(.+)$/.exec(t);
+      if (m) dimensions[m[1].trim()] = m[2].split(',').map((v) => v.trim()).filter(Boolean);
+      else if (t === '') inDimensions = false;
+    }
+  }
+  const combination = Object.keys(dimensions).length > 0;
+  const [header, ...rows] = grids[0] ?? [[]];
+  const objects = rows.map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ''])));
+  let count;
+  if (combination) {
+    const effective = cover ?? 'combination';
+    count = effective === 'pair' ? pairs(dimensions).length : Object.values(dimensions).reduce((n, v) => n * v.length, 1);
+  } else {
+    count = rows.length * Math.max(0, header.length - 1);
+  }
+  return {
+    name, title, line, kind: combination ? 'combination' : 'rule', header, rows: objects, grids,
+    dimensions, cover: combination ? (cover ?? 'combination') : null, count,
+    fingerprint: fingerprintOf(text.filter((t) => t.trim()).join('\n')),
+  };
+}
+
+// ---------------------------------------------------------------- the reader
+
+// Reads scenario text. Returns the parts, and `problems`: format faults, each
+// with a line. The link check reports every problem.
+export function parseScenario(source, file = 'scenario.md') {
+  const text = String(source).replace(/\r\n?/g, '\n');
+  const lines = text.split('\n');
+  const problems = [];
+  const problem = (line, message) => problems.push({ line, message });
+  const out = {
+    file, id: null, title: '', fields: {}, fieldLine: () => null, through: [], start: [], steps: [], assertions: [],
+    tables: new Map(), sections: new Map(), problems, sha256: sha256(text), method: FINGERPRINT_METHOD,
+  };
+
+  if (lines[0] !== '---') { problem(1, 'a scenario starts with a front matter block (---)'); return out; }
+  const end = lines.indexOf('---', 1);
+  if (end < 0) { problem(1, 'the front matter block has no closing ---'); return out; }
+  try {
+    const parsed = parseYaml(lines.slice(1, end).join('\n'));
+    out.fields = parsed.value && typeof parsed.value === 'object' && !Array.isArray(parsed.value) ? parsed.value : {};
+    out.fieldLine = (key) => { const n = parsed.lineOf([key]); return n === null ? 1 : n + 1; };
+  } catch (error) {
+    if (!(error instanceof YamlError)) throw error;
+    problem(1 + (error.line ?? 0), `the front matter cannot be read: ${error.message.replace(/ \(line \d+\)$/, '')}`);
+  }
+  const f = out.fields;
+  out.id = typeof f.id === 'string' ? f.id : null;
+  if (!out.id) problem(2, 'the front matter has no id');
+  else if (!/^[a-z0-9][a-z0-9-]*$/.test(out.id)) problem(out.fieldLine('id'), `the id "${out.id}" must be lower case letters, digits and hyphens`);
+  const list = (key) => (Array.isArray(f[key]) ? f[key].map(String) : f[key] === undefined ? [] : [String(f[key])]);
+  out.through = list('through');
+  if (!out.through.length) problem(out.fieldLine('through'), 'the front matter has no through: the ways in that the steps use');
+  if (!list('runs-in').length) problem(out.fieldLine('runs-in'), 'the front matter has no runs-in');
+  if (f.limit === undefined) problem(out.fieldLine('limit'), 'the front matter has no limit');
+  else { try { parseLimit(f.limit); } catch (e) { problem(out.fieldLine('limit'), e.message); } }
+
+  // Sections.
+  let section = null;
+  let title = '';
+  let tableBlock = null;
+  const sectionLines = new Map();
+  const flushTable = () => {
+    if (!tableBlock) return;
+    if (out.tables.has(tableBlock.name)) problem(tableBlock.line, `the case table ${tableBlock.name} is named twice`);
+    else {
+      const table = readTable(tableBlock.name, tableBlock.title, tableBlock.line, tableBlock.lines);
+      if (!table.rows.length && table.kind === 'rule') problem(table.line, `the case table ${table.name} has no rows`);
+      out.tables.set(table.name, table);
+    }
+    tableBlock = null;
+  };
+  for (let i = end + 1; i < lines.length; i += 1) {
+    const raw = lines[i];
+    const number = i + 1;
+    if (/^# /.test(raw) && !title) { title = raw.slice(2).trim(); continue; }
+    const heading = /^##\s+(.+?)\s*$/.exec(raw);
+    if (heading && !raw.startsWith('###')) {
+      flushTable();
+      section = heading[1].toLowerCase();
+      if (!sectionLines.has(section)) sectionLines.set(section, []);
+      out.sections.set(section, number);
+      continue;
+    }
+    if (section === null) continue;
+    sectionLines.get(section).push({ text: raw, number });
+    if (section === 'cases') {
+      const t = TABLE_HEADING.exec(raw);
+      if (t) { flushTable(); tableBlock = { name: t[1], title: t[2], line: number, lines: [] }; continue; }
+      if (tableBlock) tableBlock.lines.push({ text: raw, number });
+    }
+  }
+  flushTable();
+  out.title = title || (typeof f.title === 'string' ? f.title : '');
+
+  // Items with continuation lines: a marker line, then indented lines.
+  const items = (name, marker) => {
+    const result = [];
+    for (const { text: raw, number } of sectionLines.get(name) ?? []) {
+      if (!raw.trim()) continue;
+      const m = marker.exec(raw);
+      if (m) result.push({ line: number, body: raw, marker: m, text: raw.slice(m[0].length).trim() });
+      else if (/^\s+\S/.test(raw) && result.length) { const last = result.at(-1); last.text = `${last.text} ${raw.trim()}`; last.body += `\n${raw}`; }
+      else result.push({ line: number, body: raw, marker: null, text: raw.trim() });
+    }
+    return result;
+  };
+
+  for (const item of items('start with', /^-\s+/)) {
+    if (!item.marker) { problem(item.line, 'a "Start with" line must be a list item (- ...)'); continue; }
+    const defined = /^`([^`]+)`\s*:/.exec(item.text);
+    out.start.push({ line: item.line, text: item.text, name: defined ? defined[1] : null, names: names(item.text) });
+  }
+
+  const seenSteps = new Set();
+  for (const item of items('steps', /^(\d+)\.\s+/)) {
+    if (!item.marker) { problem(item.line, 'a step must be a numbered item (1. ...)'); continue; }
+    const n = Number(item.marker[1]);
+    if (seenSteps.has(n)) problem(item.line, `step ${n} is written twice`);
+    seenSteps.add(n);
+    const persona = /^As `([^`]+)`,\s*(?:through `([^`]+)`,\s*)?/i.exec(item.text);
+    const keeps = [...item.text.matchAll(/\bKeep\b[^.`]*?\bas `([^`]+)`/gi)].map((m) => m[1]);
+    const isWait = /^Wait until\b/i.test(item.text);
+    const limit = isWait ? WAIT_LIMIT.exec(item.text) : null;
+    out.steps.push({
+      n, line: item.line, text: item.text, names: names(item.text),
+      persona: persona ? persona[1] : null, way: persona?.[2] ?? null, keeps,
+      wait: isWait ? { limit: limit ? limit[0].replace(/^give up after\s+/i, '') : null } : null,
+    });
+  }
+  const ordered = out.steps.every((s, i) => s.n === i + 1);
+  if (!ordered && out.steps.length) problem(out.steps[0].line, 'the steps must be numbered 1, 2, 3 ... in order');
+
+  const seen = new Set();
+  for (const item of items('assertions', /^(?=-)/)) {
+    const flat = `${item.body.split('\n')[0]}${item.body.includes('\n') ? ` ${item.body.split('\n').slice(1).map((s) => s.trim()).join(' ')}` : ''}`;
+    const m = ASSERTION.exec(flat);
+    if (!m) { problem(item.line, `an assertion line must read "- eN exact: words" or "- eN judged: words"; found "${flat.slice(0, 60)}"`); continue; }
+    if (seen.has(m[1])) problem(item.line, `the assertion ${m[1]} is written twice`);
+    seen.add(m[1]);
+    out.assertions.push({ id: m[1], kind: m[2], gate: m[3] === 'gate', words: m[4].trim(), line: item.line, names: names(m[4]), fp: null });
+  }
+  if (!out.assertions.length) problem(out.sections.get('assertions') ?? 1, 'the scenario has no assertions');
+  else if (!out.sections.has('assertions')) problem(1, 'the scenario has no "## Assertions" section');
+  return out;
+}
+
+export function readScenario(path) {
+  return parseScenario(readFileSync(path, 'utf8'), String(path));
+}
+
+// ---------------------------------------------------------------- fingerprints
+
+const runsIn = (scenario) => [].concat(scenario.fields['runs-in'] ?? []).map(String);
+
+// Sets `fp` and `fullId` on every assertion. `tests` is the parsed value of
+// tests.yaml. Returns the scenario.
+export function addFingerprints(scenario, tests) {
+  const start = new Map(scenario.start.filter((s) => s.name).map((s) => [s.name, s]));
+  const kept = new Map();
+  for (const step of scenario.steps) for (const k of step.keeps) if (!kept.has(k)) kept.set(k, step);
+  const steps = new Map(scenario.steps.map((s) => [s.n, s]));
+  const setups = tests?.setups ?? {};
+  const fixtures = tests?.fixtures ?? {};
+  const data = tests?.data ?? {};
+  const stepRefs = (text) => [...String(text).matchAll(/\bsteps?\s+(\d+)/gi)].map((m) => Number(m[1]));
+
+  for (const a of scenario.assertions) {
+    const parts = new Map();
+    const visited = new Set();
+    const queue = [];
+    const visit = (name) => {
+      if (start.has(name)) {
+        if (!parts.has(`start ${name}`)) { parts.set(`start ${name}`, normalize(start.get(name).text)); queue.push(...start.get(name).names); }
+      } else if (kept.has(name)) visitStep(kept.get(name).n);
+      if (Object.hasOwn(setups, name)) parts.set(`setup ${name}`, JSON.stringify(sortKeys(setups[name])));
+      if (Object.hasOwn(fixtures, name)) parts.set(`fixture ${name}`, JSON.stringify(sortKeys(fixtures[name])));
+      if (Object.hasOwn(data, name)) parts.set(`data ${name}`, JSON.stringify(sortKeys(data[name])));
+    };
+    const visitStep = (n) => {
+      const step = steps.get(n);
+      if (!step || visited.has(n)) return;
+      visited.add(n);
+      parts.set(`step ${String(n).padStart(4, '0')}`, normalize(step.text));
+      queue.push(...step.names);
+    };
+    queue.push(...a.names);
+    for (const n of stepRefs(a.words)) visitStep(n);
+    const done = new Set();
+    while (queue.length) {
+      const name = queue.shift();
+      if (done.has(name)) continue;
+      done.add(name);
+      visit(name);
+    }
+    const lines = [...parts].sort(([x], [y]) => (x < y ? -1 : 1)).map(([k, v]) => `${k}=${v}`);
+    lines.push(`through=${[...scenario.through].sort().join(',')}`, `runs-in=${runsIn(scenario).sort().join(',')}`);
+    a.fp = fingerprintOf([a.words, ...lines].join('\n'));
+    a.fullId = `${scenario.id}/${a.id}#${a.fp}`;
+  }
+  return scenario;
+}
+
+// The fingerprint of a whole scenario file is its sha256. Tables carry their own.
+export function tableFingerprints(scenario) {
+  return Object.fromEntries([...scenario.tables].map(([name, t]) => [name, t.fingerprint]));
+}

package/tests/stand-in.mjs

@@ -0,0 +1,74 @@
+// Stand-ins: a local replacement for an outside service.
+//
+// The protocol between the library and a stand-in, in three lines:
+//   1. The library picks a free port. It gives the port to the stand-in as the
+//      PORT variable and as ATLAS_STAND_IN_PORT, and fills {port} in the
+//      start command. The stand-in listens on 127.0.0.1 at that port.
+//   2. GET /__atlas/calls answers 200 with {"calls": [ ... ]}: one entry for
+//      each call that the product made. Calls to /__atlas/* do not count.
+//      A call from the library carries the header `x-atlas-library: 1` (a
+//      ready poll, a calls read, any call of the kit). It does not count
+//      either. Only a call without that header can be the first call.
+//   3. Before step 1 the library reads that list. An empty list means the
+//      product never called the stand-in, and the run is `blocked`.
+// `serveStandIn` does all three. A repo writes only the handler.
+import { createServer } from 'node:http';
+
+export const CALLS_PATH = '/__atlas/calls';
+// Every call that the library makes to a stand-in carries this header.
+export const LIBRARY_HEADER = 'x-atlas-library';
+
+// handler({ method, path, query, headers, body, raw }) returns
+// { status = 200, body = null, headers = {} }.
+export function createStandIn(handler) {
+  const calls = [];
+  const server = createServer((req, res) => {
+    const chunks = [];
+    req.on('data', (chunk) => chunks.push(chunk));
+    req.on('end', async () => {
+      const url = new URL(req.url, 'http://stand-in.local');
+      if (url.pathname === CALLS_PATH) {
+        res.writeHead(200, { 'content-type': 'application/json' });
+        res.end(JSON.stringify({ calls }));
+        return;
+      }
+      const raw = Buffer.concat(chunks).toString('utf8');
+      let body = raw;
+      try { body = raw ? JSON.parse(raw) : null; } catch { /* keep the text */ }
+      if (req.headers[LIBRARY_HEADER] !== '1' && !url.pathname.startsWith('/__atlas/')) calls.push({ method: req.method, path: url.pathname, at: new Date().toISOString() });
+      try {
+        const answer = (await handler({ method: req.method, path: url.pathname, query: Object.fromEntries(url.searchParams), headers: req.headers, body, raw, calls })) ?? {};
+        const payload = answer.body === undefined || answer.body === null ? '' : (typeof answer.body === 'string' ? answer.body : JSON.stringify(answer.body));
+        res.writeHead(answer.status ?? 200, { ...(payload && typeof answer.body !== 'string' ? { 'content-type': 'application/json' } : {}), ...(answer.headers ?? {}) });
+        res.end(payload);
+      } catch (error) {
+        res.writeHead(500, { 'content-type': 'application/json' });
+        res.end(JSON.stringify({ error: String(error.message) }));
+      }
+    });
+  });
+  return {
+    calls,
+    server,
+    listen: (port = 0, host = '127.0.0.1') => new Promise((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(port, host, () => resolve(`http://${host}:${server.address().port}`));
+    }),
+    close: () => new Promise((resolve) => { server.close(() => resolve()); server.closeAllConnections?.(); }),
+  };
+}
+
+// For a stand-in that the library starts as a process: listens on the port that the library gives.
+export async function serveStandIn(handler, { port = Number(process.env.ATLAS_STAND_IN_PORT ?? process.env.PORT) } = {}) {
+  if (!Number.isInteger(port) || port <= 0) throw new Error('no port: the library gives the port in ATLAS_STAND_IN_PORT');
+  const standIn = createStandIn(handler);
+  await standIn.listen(port);
+  return standIn;
+}
+
+// How many calls a stand-in reports. Accepts a list, a number, or {calls} / {count}.
+export function countCalls(body) {
+  const value = body && typeof body === 'object' && !Array.isArray(body) ? (body.calls ?? body.count) : body;
+  if (Array.isArray(value)) return value.length;
+  return Number.isFinite(Number(value)) ? Number(value) : 0;
+}