@arjunkhera/atlas 0.3.41 → 0.3.43

package/tests/contract.py

@@ -0,0 +1,893 @@
+"""The contract library for Python (design section 8.1). It is the Python form of
+contract.mjs, on the standard unittest runner, so a repo needs no new package.
+
+    # in test/scenarios/routing_score_test.py, with the kit copy in test/atlas/
+    import sys
+    from pathlib import Path
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "atlas"))
+    from contract import scenario
+
+    @scenario("../../scenarios/routing-score.md", adapter=adapter)
+    def RoutingScore(run, way):
+        owner = run.persona("owner", way)
+        # ... steps, through the actions of the area ...
+        with run.gate("routing-score/e1#e46b3c"):
+            assert first.exit == 0, first.stderr
+        with run.check("routing-score/e2#7d78e1") as proof:
+            assert "held-out: 3/3 routed correctly" in right.stdout
+            proof.value = right.stdout.strip()
+
+What `scenario(path, adapter=...)` does
+  1. It reads the scenario and atlas/tests.yaml through the bridge
+     (bridge.mjs), so the words, the fingerprints, the tests.yaml check and
+     the guard parts are read by the same code as the link check. It needs
+     node, as the link check does. It refuses to run when either file has a
+     fault: the import fails.
+  2. It makes one unittest.TestCase class, with one test for each way in of
+     `through:`. Each test runs the body with (run, way).
+  3. It writes the evidence file, schema `evidence: 1`, the same file as
+     evidence.mjs writes: at the start and after each assertion. So the run
+     half of the link check reads a Python run as it reads a JavaScript run.
+  4. A test fails when any assertion is not `pass`. That holds for `fail`,
+     `not checked`, `not exercised`, `blocked` and `unsure`.
+  5. When the scenario's `runs-in` excludes the environment, each result is
+     `not here` and the test passes.
+
+A check is a `with` block. The block must hold an assert statement (the
+Python lint of the link check refuses one that does not). A failed assert,
+or any other error in the block, is a `fail`, and the run goes on. A failed
+gate stops the run; later assertions read `not checked`. A block that is not
+entered checks nothing, so its assertion stays `not checked`.
+
+What this library does not do yet. Each one ends `blocked`, with the reason,
+and never a pass: an environment that starts a product or a stand-in, run
+secrets, a persona with a secret, `make:` or `sign-in:` that the adapter does
+not give, `repeat`, judged assertions, model calls, and the drivers of the
+kit. A way in gets its driver from the adapter; `child_env` here is the guard
+that a driver gives its child process.
+
+The adapter binds the library to one product. Every part is optional:
+  drivers   {way or driver name: (ctx) -> driver}; ctx holds way, spec, record,
+            redactor, root, run_id and tests. driver.call(request, **opts) and
+            driver.close()
+  warm_up   (env) -> None, once before the first test; raise Blocked when the
+            product cannot run on this machine
+  actions   an object, or (run) -> object: the verbs of the area, on run.actions
+  data      {rule: (run_id, count, rule) -> value}, a data rule of the repo
+  makers, sign_ins  {name: (ctx) -> {credential, headers, cleanup}}
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import inspect
+import json
+import os
+import re
+import secrets as _secrets
+import signal
+import subprocess
+import sys
+import threading
+import time
+import types
+import unittest
+from pathlib import Path
+
+__all__ = [
+    "scenario", "define_adapter", "wait_until", "parse_limit", "child_env",
+    "Blocked", "GateFailed", "WaitFailed", "WaitTimeout", "Redactor", "RUN_ID",
+    "EVIDENCE_SCHEMA", "RESULTS",
+]
+
+KIT = Path(__file__).resolve().parent
+BRIDGE = KIT / "bridge.mjs"
+FULL_ID = re.compile(r"^(.+)/(e\d+)#([0-9a-f]{6})$")
+EVIDENCE_SCHEMA = 1
+RESULTS = ("pass", "fail", "not checked", "not exercised", "blocked", "not here", "flaky", "unsure")
+MAX_BODY = 8000
+
+
+# ---------------------------------------------------------------- errors (errors.mjs)
+
+class Blocked(Exception):
+    """A setup failed, a guard refused, or a precondition does not hold. The result is `blocked`."""
+
+
+class GateFailed(Exception):
+    """A failed gate. It stops the body of the test."""
+
+
+class WaitFailed(Exception):
+    """A wait met its fail state. This is a product result, so it is a `fail`."""
+
+    def __init__(self, reason, last=None):
+        super().__init__(reason)
+        self.last = last
+
+
+class WaitTimeout(Exception):
+    """A wait ran out of time. This is a product result, so it is a `fail`."""
+
+    def __init__(self, reason, last=None):
+        super().__init__(reason)
+        self.last = last
+
+
+class _OutOfTime(BaseException):
+    """Not an Exception, so neither a check block nor an `except Exception` of a test can swallow it."""
+
+
+def define_adapter(**parts):
+    return types.SimpleNamespace(**parts)
+
+
+# ---------------------------------------------------------------- the run id and fresh values (fresh.mjs)
+
+def make_run_id(env=os.environ):
+    if env.get("ATLAS_RUN_ID"):
+        return env["ATLAS_RUN_ID"]
+    if env.get("GITHUB_RUN_ID"):
+        return f"ci{env['GITHUB_RUN_ID']}-{env.get('GITHUB_RUN_ATTEMPT', 1)}"
+    return f"run{_secrets.token_hex(3)}"
+
+
+RUN_ID = make_run_id()
+
+_JAN1 = _dt.date(2035, 1, 1)
+_FIRST_MONDAY = _JAN1 + _dt.timedelta(days=((8 - (_JAN1.isoweekday() % 7)) % 7) or 7)
+_SPREAD_WEEKS = 520
+
+
+def monday_for(run_id, count):
+    digest = hashlib.sha256(f"{run_id}:{count}".encode()).digest()
+    weeks = int.from_bytes(digest[:4], "big") % _SPREAD_WEEKS
+    return (_FIRST_MONDAY + _dt.timedelta(weeks=weeks)).isoformat()
+
+
+def make_fresh(run_id, rules, adapter_rules=None):
+    rules = rules or {}
+    adapter_rules = adapter_rules or {}
+    counts = {}
+    made = []
+
+    def fresh(rule):
+        if rule not in rules and rule not in adapter_rules:
+            raise ValueError(f'there is no data rule "{rule}" in tests.yaml')
+        count = counts.get(rule, 0) + 1
+        counts[rule] = count
+        if rule in adapter_rules:
+            value = adapter_rules[rule](run_id, count, rule)
+        elif re.search("monday", rule, re.I):
+            value = monday_for(run_id, f"{rule}:{count}")
+        else:
+            value = f"{re.sub('^fresh-', '', rule)}-{run_id}-{count}"
+        made.append({"rule": rule, "value": value})
+        return value
+
+    fresh.made = made
+    return fresh
+
+
+# ---------------------------------------------------------------- redaction (redact.mjs)
+
+MIN_SECRET_LENGTH = 4
+_SHAPES = [
+    (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), "[redacted:jwt]"),
+    (re.compile(r"\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}", re.I), r"\1 [redacted]"),
+    (re.compile(r"([?&](?:X-Amz-Signature|X-Amz-Credential|X-Amz-Security-Token|Signature|sig|access_token)=)[^&\s\"']+", re.I), r"\1[redacted]"),
+]
+
+
+class Redactor:
+    """Redaction by value. No value in the set reaches an evidence file or a log."""
+
+    def __init__(self, values=()):
+        self.values = set()
+        self.add(values)
+
+    def add(self, value):
+        for one in value if isinstance(value, (list, tuple, set)) else [value]:
+            if isinstance(one, str) and len(one) >= MIN_SECRET_LENGTH:
+                self.values.add(one)
+        return self
+
+    def text(self, value):
+        if not isinstance(value, str):
+            return value
+        out = value
+        for secret in sorted(self.values, key=len, reverse=True):
+            out = out.replace(secret, "[redacted]")
+        for shape, replacement in _SHAPES:
+            out = shape.sub(replacement, out)
+        return out
+
+    def deep(self, value):
+        if isinstance(value, str):
+            return self.text(value)
+        if isinstance(value, (list, tuple)):
+            return [self.deep(one) for one in value]
+        if isinstance(value, BaseException):
+            return self.text(str(value))
+        if isinstance(value, dict):
+            return {self.text(str(k)): self.deep(v) for k, v in value.items()}
+        return value
+
+
+def clip(value, redactor=None):
+    """Redact first, then cut: a cut before the redaction could split a secret."""
+    safe = redactor.deep(value) if redactor else value
+    text = safe if isinstance(safe, str) else json.dumps(safe, separators=(",", ":"), default=str)
+    if len(text) <= MAX_BODY:
+        return safe
+    return f"{text[:MAX_BODY]}... [{len(text) - MAX_BODY} more characters]"
+
+
+# ---------------------------------------------------------------- the child environment (guards.mjs)
+
+_ALLOWED = re.compile(r"^(?:PATH|HOME|LANG|LC_[A-Z_]+|TMPDIR|TERM)$")
+_CREDENTIAL_NAME = re.compile(r"TOKEN|AUTH|KEY|SECRET|PASSWORD|PASS|CRED", re.I)
+_LOOPBACK_PROXY = re.compile(r"^http://(127\.0\.0\.1|localhost|\[::1\]):\d{1,5}/?$")
+
+
+def child_env(parent=None, values=None):
+    """The environment of a child process: the kit's allow-list from `parent`, then `values` on top.
+    The rule is childEnv of guards.mjs, word for word."""
+    parent = os.environ if parent is None else parent
+    out = {}
+    for name, value in parent.items():
+        if value is None:
+            continue
+        if _ALLOWED.match(name) or (name.startswith("NODE_") and name != "NODE_TEST_CONTEXT" and not _CREDENTIAL_NAME.search(name)):
+            out[name] = value
+    if parent.get("CLAUDE_CODE_REMOTE") == "true":
+        proxies = [n for n in ("HTTPS_PROXY", "https_proxy") if parent.get(n) is not None]
+        if proxies and all(_LOOPBACK_PROXY.match(parent[n]) for n in proxies):
+            for name in proxies + ["NO_PROXY", "no_proxy"]:
+                if parent.get(name) is not None:
+                    out[name] = parent[name]
+    for name, value in (values or {}).items():
+        if value is not None:
+            out[name] = str(value)
+    return out
+
+
+# ---------------------------------------------------------------- waits (wait.mjs)
+
+_UNITS = {"ms": 1, "millisecond": 1, "milliseconds": 1, "s": 1000, "second": 1000, "seconds": 1000,
+          "m": 60000, "min": 60000, "minute": 60000, "minutes": 60000, "h": 3600000, "hour": 3600000, "hours": 3600000}
+
+
+def parse_limit(limit):
+    """"30s", "2m", "2 minutes" or a number of milliseconds. Returns milliseconds."""
+    if isinstance(limit, (int, float)) and not isinstance(limit, bool) and limit > 0:
+        return limit
+    m = re.match(r"^(\d+(?:\.\d+)?)\s*([a-z]+)$", str(limit if limit is not None else "").strip(), re.I)
+    if not m or m.group(2).lower() not in _UNITS:
+        raise ValueError(f'a limit is a number of milliseconds, or like "30s" or "2m"; found "{limit}"')
+    return float(m.group(1)) * _UNITS[m.group(2).lower()]
+
+
+def _show(last):
+    body = last.get("body", last) if isinstance(last, dict) else getattr(last, "body", last)
+    return json.dumps(body, default=str)[:600]
+
+
+def wait_until(read, until, fail=None, absent=None, limit=None, every="250ms", absent_limit=10):
+    """Wait for a state, never for a fixed time. A wait with no limit is refused."""
+    if not callable(read):
+        raise ValueError("wait_until needs a function that reads the state")
+    if not callable(until):
+        raise ValueError("wait_until needs an `until` state")
+    if limit is None:
+        raise ValueError("wait_until needs a `limit`: a wait with no time limit is refused")
+    deadline = time.monotonic() + parse_limit(limit) / 1000
+    absent_reads = 0
+    while True:
+        last = read()
+        if fail and fail(last):
+            raise WaitFailed(f"the wait met its fail state: {_show(last)}", last)
+        if until(last):
+            return last
+        absent_reads = absent_reads + 1 if absent and absent(last) else 0
+        if absent_reads >= absent_limit:
+            raise WaitFailed(f"the field that the wait reads was absent for {absent_reads} reads in a row", last)
+        if time.monotonic() >= deadline:
+            raise WaitTimeout(f"the wait ran out after {limit}; last state: {_show(last)}", last)
+        time.sleep(parse_limit(every) / 1000)
+
+
+# ---------------------------------------------------------------- the evidence (evidence.mjs)
+
+def _now():
+    return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _commit_of(root):
+    env = os.environ
+    if env.get("GITHUB_HEAD_SHA"):
+        return env["GITHUB_HEAD_SHA"]
+    if env.get("GITHUB_EVENT_PATH"):
+        try:
+            sha = json.loads(Path(env["GITHUB_EVENT_PATH"]).read_text()).get("pull_request", {}).get("head", {}).get("sha")
+            if sha:
+                return sha
+        except (OSError, ValueError, AttributeError):
+            pass
+    if env.get("GITHUB_SHA"):
+        return env["GITHUB_SHA"]
+    try:
+        r = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=10)
+        return r.stdout.strip() if r.returncode == 0 else None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+class Evidence:
+    """One evidence file for each run and scenario. It is claimed with an exclusive create, so no
+    earlier run is overwritten. Every string is redacted by value before it is written."""
+
+    def __init__(self, evidence_dir, run_id, environment, sc, root, redactor):
+        self.dir = Path(evidence_dir) / run_id
+        self.redactor = redactor
+        self.record = {
+            "evidence": EVIDENCE_SCHEMA, "run_id": run_id, "environment": environment,
+            "scenario": sc["id"], "title": sc["title"],
+            "source": os.path.relpath(sc["file"], root).replace(os.sep, "/"), "source_sha256": sc["sha256"],
+            "fingerprint_method": sc["method"],
+            "commit": _commit_of(root), "started": _now(), "ended": None, "verdict": "not finished",
+            "ways": {}, "assertions": [], "tables": {}, "fresh": [], "calls": [], "notes": [], "files": [],
+            "provided_secrets": [], "fault_run": os.environ.get("ATLAS_FAULT_RUN") or None,
+            "language": "python",
+        }
+        for way in sc["ways"]:
+            self.record["ways"][way] = {"verdict": "not finished", "reason": None, "started": None, "ended": None}
+        for a in sc["assertions"]:
+            for way in a["ways"]:
+                self.record["assertions"].append({"id": a["fullId"], "way": way, "result": "not checked", "proof": None})
+        self.dir.mkdir(parents=True, exist_ok=True)
+        self.path = self._claim(sc["id"])
+        self.write()
+
+    def _claim(self, scenario_id):
+        n = 1
+        while True:
+            path = self.dir / (f"{scenario_id}.json" if n == 1 else f"{scenario_id}.{n}.json")
+            try:
+                os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
+                return path
+            except FileExistsError:
+                n += 1
+
+    def entry(self, full_id, way):
+        for a in self.record["assertions"]:
+            if a["id"] == full_id and a["way"] == way:
+                return a
+        raise ValueError(f'the evidence has no assertion {full_id} for the way in "{way}"')
+
+    def set_result(self, full_id, way, result, proof=None):
+        if result not in RESULTS:
+            raise ValueError(f'"{result}" is not a result')
+        entry = self.entry(full_id, way)
+        entry["result"] = result
+        entry["proof"] = {k: v for k, v in (proof or {}).items() if v is not None} or None
+        self.write()
+
+    def set_way(self, way, **patch):
+        self.record["ways"][way].update(patch)
+        verdicts = [w["verdict"] for w in self.record["ways"].values()]
+        if "fail" in verdicts:
+            self.record["verdict"] = "fail"
+        elif "blocked" in verdicts:
+            self.record["verdict"] = "blocked"
+        elif all(v in ("pass", "not here") for v in verdicts):
+            self.record["verdict"] = "not here" if all(v == "not here" for v in verdicts) else "pass"
+        else:
+            self.record["verdict"] = "not finished"
+        if "not finished" not in verdicts:
+            self.record["ended"] = _now()
+        self.write()
+
+    def add_call(self, call):
+        n = len(self.record["calls"]) + 1
+        self.record["calls"].append({"n": n, **call})
+        return n
+
+    def add_file(self, path):
+        self.record["files"].append(os.path.relpath(path, self.dir).replace(os.sep, "/"))
+        self.write()
+
+    def note(self, text):
+        self.record["notes"].append(str(text))
+        self.write()
+
+    def write(self):
+        safe = self.redactor.deep(self.record)
+        temp = self.path.with_name(self.path.name + ".tmp")
+        temp.write_text(json.dumps(safe, indent=2, ensure_ascii=False) + "\n")
+        os.replace(temp, self.path)
+
+
+# ---------------------------------------------------------------- reading the scenario
+
+def _load(file):
+    try:
+        r = subprocess.run(["node", str(BRIDGE), "load", str(file)], capture_output=True, text=True, timeout=60)
+    except FileNotFoundError as error:
+        raise RuntimeError("the Python contract library needs node to read a scenario (the bridge, bridge.mjs)") from error
+    try:
+        data = json.loads(r.stdout)
+    except ValueError as error:
+        raise RuntimeError(f"the bridge gave no answer: {r.stderr.strip()[:400]}") from error
+    if "error" in data:
+        raise RuntimeError(data["error"])
+    return data
+
+
+def _caller_dir():
+    for frame in inspect.stack()[2:]:
+        if Path(frame.filename).resolve() != Path(__file__).resolve():
+            return Path(frame.filename).resolve().parent
+    return Path.cwd()
+
+
+def _as_list(x):
+    if isinstance(x, list):
+        return [str(v) for v in x]
+    return [] if x is None else [str(x)]
+
+
+def _one_line(text):
+    return re.sub(r"\s+", " ", str(text or "")).strip()
+
+
+# ---------------------------------------------------------------- the check block
+
+class _Check:
+    def __init__(self, run, full, mode):
+        self.run = run
+        self.full = full
+        self.mode = mode
+        self.value = None
+
+    def __enter__(self):
+        self.run._open(f"run.{self.mode}")
+        self.found = self.run._find(self.full)
+        return self
+
+    def __exit__(self, kind, error, trace):
+        # A block or a stopped gate inside the block is no product result of this check: it goes on up.
+        if kind is not None and (not issubclass(kind, Exception) or issubclass(kind, (Blocked, GateFailed))):
+            return False
+        a, fp = self.found
+        failure = None
+        if fp != a["fp"]:
+            failure = f"the fingerprint {fp} is old: the words of {a['id']} now print {a['fp']}; convert the test again"
+        elif error is not None:
+            message = str(error)
+            if isinstance(error, AssertionError):
+                at = trace.tb_lineno if trace else "?"
+                while trace is not None and trace.tb_next is not None:
+                    trace = trace.tb_next
+                    at = trace.tb_lineno
+                failure = message or f"an assert statement failed (line {at})"
+            else:
+                failure = f"{type(error).__name__}: {message}" if message else type(error).__name__
+        self.run._settle_one(a, failure, self.value)
+        if failure and self.mode == "gate":
+            raise GateFailed(f"[{a['id']}] {failure}")
+        return True
+
+
+# ---------------------------------------------------------------- the run object
+
+class Run:
+    def __init__(self, ctx, state, way, ev, adapter):
+        self._ctx = ctx
+        self._state = state
+        self._ev = ev
+        self._adapter = adapter
+        self._closed = False
+        self._mark = len(ev.record["calls"])
+        sc = ctx["scenario"]
+        self.id = sc["id"]
+        self.title = sc["title"]
+        self.run_id = RUN_ID
+        self.way = way
+        self.scenario = sc
+        self.tests = ctx["tests"]
+        self.root = ctx["root"]
+        self.round = None
+        self.cleanups = []
+        if state.get("fresh") is None:
+            state["fresh"] = make_fresh(RUN_ID, self.tests.get("data"), getattr(adapter, "data", None))
+        actions = getattr(adapter, "actions", None)
+        actions = actions(self) if callable(actions) else (actions or {})
+        self.actions = types.SimpleNamespace(**actions) if isinstance(actions, dict) else actions
+
+    # -- internal
+
+    def _open(self, what):
+        if self._closed:
+            raise RuntimeError(f"the run is closed: {what} came after the scenario ended, so it changed nothing")
+
+    def _calls_since(self):
+        start = self._mark
+        self._mark = len(self._ev.record["calls"])
+        return [c["n"] for c in self._ev.record["calls"][start:]]
+
+    def _find(self, full):
+        sc = self.scenario
+        m = FULL_ID.match(str(full))
+        if not m:
+            raise ValueError(f'"{full}" is not a full assertion id like {sc["id"]}/e1#abc123')
+        if m.group(1) != sc["id"]:
+            raise ValueError(f'"{full}" names the scenario "{m.group(1)}"; this is "{sc["id"]}"')
+        for a in sc["assertions"]:
+            if a["id"] == m.group(2):
+                return a, m.group(3)
+        raise ValueError(f"the scenario has no assertion {m.group(2)}")
+
+    def _target(self, a):
+        return self.way if self.way in a["ways"] else a["ways"][0]
+
+    def _settle_one(self, a, failure, value):
+        if a["kind"] == "judged":
+            failure = failure or f"{a['id']} is a judged assertion; the Python library has no judge yet"
+        target = self._target(a)
+        entry = self._ev.entry(a["fullId"], target)
+        calls = self._calls_since()
+        earlier = entry if (entry.get("proof") or {}).get("settled_by") not in (None, self.way) else None
+        also = ([*earlier["proof"].get("also", []), {"run": earlier["proof"]["settled_by"], "result": earlier["result"]}]
+                if earlier else (entry.get("proof") or {}).get("also"))
+        if failure:
+            self._ev.set_result(a["fullId"], target, "fail", {"error": self.redact(failure), "calls": calls, "settled_by": self.way, "also": also})
+        elif entry["result"] != "fail":
+            self._ev.set_result(a["fullId"], target, "pass", {"value": None if value is None else clip(value, self._state["redactor"]), "calls": calls, "settled_by": self.way, "also": also})
+        else:
+            entry["proof"] = {**(entry.get("proof") or {}), "also": [*(entry["proof"] or {}).get("also", []), {"run": self.way, "result": "pass"}]}
+            self._ev.write()
+
+    def _record(self, call):
+        if self._closed:
+            return None
+        return self._ev.add_call(self._state["redactor"].deep(call))
+
+    def close(self):
+        self._closed = True
+
+    # -- the parts of section 8.1
+
+    def check(self, full):
+        """One assertion, as a `with` block. A failure is recorded and the run goes on."""
+        return _Check(self, full, "check")
+
+    def gate(self, full):
+        """The same, and a failure stops the run. Later assertions read `not checked`."""
+        return _Check(self, full, "gate")
+
+    def not_exercised(self, full, why):
+        """A "when" assertion whose situation never happened in this run."""
+        self._open("run.not_exercised")
+        a, fp = self._find(full)
+        target = self._target(a)
+        if fp != a["fp"]:
+            self._settle_one(a, f"the fingerprint {fp} is old: the words of {a['id']} now print {a['fp']}; convert the test again", None)
+            return
+        if self._ev.entry(a["fullId"], target)["result"] in ("pass", "fail"):
+            return
+        self._ev.set_result(a["fullId"], target, "not exercised", {"why": str(why), "calls": self._calls_since()})
+
+    def blocked(self, why):
+        """The test could not run. Never use it for a product result."""
+        raise Blocked(why)
+
+    def judge(self, full, evidence=None):
+        raise Blocked(f"{full}: the Python contract library has no judge yet; write this scenario with contract.mjs")
+
+    def _table(self, name, fingerprint=None):
+        t = self.scenario["tables"].get(name)
+        if not t:
+            raise ValueError(f"the scenario has no case table {name}")
+        self._ev.record["tables"][name] = t["fingerprint"]
+        if fingerprint and fingerprint != t["fingerprint"]:
+            raise ValueError(f"the table {name} now prints {t['fingerprint']}, not {fingerprint}; its rows changed")
+        return t
+
+    def table(self, name, fingerprint=None):
+        """The rows of a rule table, as dicts keyed by the header, read at run time."""
+        return [dict(row) for row in self._table(name, fingerprint)["rows"]]
+
+    def cells(self, name, fingerprint=None):
+        t = self._table(name, fingerprint)
+        if t["kind"] != "rule":
+            raise ValueError(f"{name} is a combination table; use run.cases")
+        first, *rest = t["header"]
+        return [{"row": row[first], "column": col, "value": row[col], "row_index": i, "label": f"{name}: {row[first]} / {col}"}
+                for i, row in enumerate(t["rows"]) for col in rest]
+
+    def cases(self, name, fingerprint=None):
+        t = self._table(name, fingerprint)
+        if t["kind"] != "combination":
+            raise ValueError(f"{name} is a rule table; use run.table or run.cells")
+        return [{**one, "label": f"{name} case {i + 1}"} for i, one in enumerate(t["cases"])]
+
+    def driver(self, way_name=None):
+        way_name = way_name or self.way
+        drivers = self._state["drivers"]
+        if way_name not in drivers:
+            spec = (self.tests.get("ways-in") or {}).get(way_name)
+            if not spec:
+                raise ValueError(f'there is no way in "{way_name}" in tests.yaml')
+            # A kit driver and the guard parts of a way in hold guards (hosts, the model meter, tools)
+            # that the Python library does not apply yet. An adapter driver must not stand in for them.
+            if spec.get("driver") in self._ctx["shippedDrivers"]:
+                raise Blocked(f'the way in "{way_name}" uses the kit driver "{spec["driver"]}", which has no Python form yet; write the test with contract.mjs')
+            guarded = [k for k in self._ctx["wayGuardParts"] if k in spec]
+            if guarded:
+                raise Blocked(f'the way in "{way_name}" sets {", ".join(guarded)}, guard parts that the Python library does not apply yet; write the test with contract.mjs')
+            given = getattr(self._adapter, "drivers", None) or {}
+            make = given.get(way_name) or given.get(spec.get("driver"))
+            if not make:
+                raise Blocked(f'the way in "{way_name}" uses the driver "{spec.get("driver")}", which has no Python form in the kit yet; give it in the adapter (drivers), or write the test with contract.mjs')
+            state = self._state
+            drivers[way_name] = make({
+                "way": way_name, "spec": spec, "root": self.root, "run_id": RUN_ID, "tests": self.tests,
+                "redactor": state["redactor"], "record": lambda call: state["live"]._record(call),
+            })
+        return drivers[way_name]
+
+    def persona(self, name, way_name=None):
+        """A test user from tests.yaml, made or loaded for this run."""
+        way_name = way_name or self.way
+        key = (name, way_name)
+        personas = self._state["personas"]
+        if key in personas:
+            return personas[key]
+        spec = (self.tests.get("personas") or {}).get(name)
+        if spec is None:
+            raise ValueError(f'there is no persona "{name}" in tests.yaml')
+        drv = self.driver(way_name)
+        extra = {}
+        if spec.get("make") or spec.get("sign-in"):
+            kind, made_by = ("makers", spec["make"]) if spec.get("make") else ("sign_ins", spec["sign-in"])
+            maker = (getattr(self._adapter, kind, None) or {}).get(made_by)
+            if not maker:
+                raise Blocked(f'persona "{name}" needs the {"maker" if kind == "makers" else "sign-in"} "{made_by}" and the adapter does not give it')
+            extra = maker({"run": self, "name": name, "spec": spec, "way": way_name, "driver": drv, "fresh": self.fresh}) or {}
+            if extra.get("cleanup"):
+                self.cleanups.append(extra["cleanup"])
+        elif spec.get("secret"):
+            raise Blocked(f'persona "{name}" needs the secret {spec["secret"]}; the Python library gives no run secrets yet')
+        credential = extra.get("credential")
+        self._state["redactor"].add(credential)
+        for value in (extra.get("headers") or {}).values():
+            self._state["redactor"].add(str(value))
+
+        def call(request=None, **opts):
+            return drv.call(request or {}, credential=credential, headers=extra.get("headers") or {}, persona=name, **opts)
+
+        persona = types.SimpleNamespace(**{k: v for k, v in extra.items() if k not in ("credential", "cleanup")},
+                                        name=name, role=spec.get("role"), way=way_name, credential=credential, call=call)
+        personas[key] = persona
+        return persona
+
+    def fresh(self, rule):
+        """A fresh value from a data rule. It carries the run id; a date is picked from it."""
+        self._open("run.fresh")
+        value = self._state["fresh"](rule)
+        self._ev.record["fresh"] = list(self._state["fresh"].made)
+        self._ev.write()
+        return value
+
+    def wait(self, read, **opts):
+        return wait_until(read, **opts)
+
+    def wait_for(self, full, read, **opts):
+        """A wait that serves one assertion: a wait that fails or runs out fails it, and stops the run."""
+        self._open("run.wait_for")
+        a, _ = self._find(full)
+        try:
+            return wait_until(read, **opts)
+        except (WaitFailed, WaitTimeout) as error:
+            self._ev.set_result(a["fullId"], self._target(a), "fail", {"error": str(error), "last": clip(error.last, self._state["redactor"])})
+            raise GateFailed(f"[{a['id']}] {error}") from error
+
+    def note(self, text):
+        self._open("run.note")
+        self._ev.note(self.redact(str(text)))
+
+    def attach(self, file_name):
+        """Keep a file as evidence. Returns the path to write to."""
+        self._open("run.attach")
+        folder = self._ev.dir / self.id
+        folder.mkdir(parents=True, exist_ok=True)
+        path = folder / file_name
+        self._ev.add_file(path)
+        return path
+
+    def on_cleanup(self, fn):
+        self.cleanups.append(fn)
+
+    def redact(self, text):
+        return self._state["redactor"].text(text)
+
+
+# ---------------------------------------------------------------- settling a way in
+
+def _settle(sc, way, ev, error):
+    """Turns the end of a body into results, a verdict and a list of problems (settle of contract.mjs)."""
+    mine = [a for a in ev.record["assertions"] if a["way"] == way or a["way"] not in sc["through"]]
+    problems = []
+    verdict, reason = "pass", None
+    if isinstance(error, Blocked):
+        verdict, reason = "blocked", str(error)
+        for a in mine:
+            if a["result"] == "not checked":
+                a["result"], a["proof"] = "blocked", {"why": reason}
+    elif isinstance(error, GateFailed):
+        verdict, reason = "fail", "a gate assertion failed, so the run stopped"
+    elif error is not None:
+        verdict, reason = "fail", f"{type(error).__name__}: {error}" if str(error) else type(error).__name__
+    for a in mine:
+        if a["result"] in ("pass", "not here"):
+            continue
+        if verdict == "pass":
+            verdict = "fail"
+        proof = a.get("proof") or {}
+        detail = proof.get("error") or proof.get("why")
+        problems.append(f"[{a['id']}] {a['result']}" + (f": {detail}" if detail else ""))
+    if verdict == "blocked":
+        problems.insert(0, f"BLOCKED: {reason}")
+    elif reason and not isinstance(error, GateFailed):
+        problems.append(f"the run stopped: {reason}")
+    ev.set_way(way, verdict=verdict, reason=reason, ended=_now())
+    for extra in [w for w in sc["ways"] if w not in sc["through"]]:
+        own = [a["result"] for a in ev.record["assertions"] if a["way"] == extra]
+        stopped = all(r in ("pass", "not here", "not checked", "blocked") for r in own) and any(r in ("not checked", "blocked") for r in own)
+        extra_reason = None
+        if "fail" in own:
+            extra_verdict, extra_reason = "fail", f"an assertion through {extra} failed"
+        elif stopped:
+            extra_verdict = "blocked"
+            extra_reason = ("the gate stopped the run" if isinstance(error, GateFailed) else reason if verdict == "blocked"
+                            else "the run stopped before every assertion through this way in was checked")
+        elif all(r in ("pass", "not here") for r in own):
+            extra_verdict = "not here" if all(r == "not here" for r in own) else "pass"
+        else:
+            extra_verdict, extra_reason = "fail", f"an assertion through {extra} is not a pass"
+        ev.set_way(extra, verdict=extra_verdict, reason=extra_reason, started=ev.record["ways"][way]["started"], ended=_now())
+    return verdict, problems
+
+
+class _Limit:
+    """The time limit of the scenario. It needs SIGALRM on the main thread; elsewhere no limit applies."""
+
+    def __init__(self, ms, text):
+        self.seconds = ms / 1000
+        self.text = text
+        self.armed = False
+
+    def __enter__(self):
+        if hasattr(signal, "setitimer") and threading.current_thread() is threading.main_thread():
+            def ring(signum, frame):
+                raise _OutOfTime(f"the scenario ran out of time: its limit is {self.text}")
+            self.before = signal.signal(signal.SIGALRM, ring)
+            signal.setitimer(signal.ITIMER_REAL, self.seconds)
+            self.armed = True
+        return self
+
+    def __exit__(self, *exc):
+        if self.armed:
+            signal.setitimer(signal.ITIMER_REAL, 0)
+            signal.signal(signal.SIGALRM, self.before)
+        return False
+
+
+def _method_name(way):
+    return "test_" + re.sub(r"\W", "_", way)
+
+
+def scenario(path, adapter=None):
+    """Decorates the body of a scenario test, (run, way) -> None, and returns a unittest.TestCase class.
+    A relative path is read from the folder of the test file."""
+    base = _caller_dir()
+    adapter = adapter or define_adapter()
+
+    def wrap(body):
+        file = Path(path) if Path(path).is_absolute() else (base / path)
+        ctx = _load(file.resolve())
+        sc = ctx["scenario"]
+        environment = ctx["environment"]
+        runs_here = environment in _as_list(sc["fields"].get("runs-in"))
+        limit_text = sc["fields"].get("limit")
+        limit_ms = parse_limit(limit_text)
+        state = {"redactor": Redactor(), "evidence": None, "drivers": {}, "personas": {}, "fresh": None, "warmed": False}
+
+        def evidence():
+            if state["evidence"] is None:
+                state["evidence"] = Evidence(ctx["evidenceDir"], RUN_ID, environment, sc, ctx["root"], state["redactor"])
+            return state["evidence"]
+
+        def ready():
+            if ctx["guardRefusal"]:
+                raise Blocked(ctx["guardRefusal"])
+            # Under -O or PYTHONOPTIMIZE, Python drops every assert statement, so every check would pass.
+            if sys.flags.optimize or not __debug__:
+                raise Blocked("Python runs with -O or PYTHONOPTIMIZE, which removes every assert statement; run it without them")
+            named = [n for n in sc.get("names", []) if n in (ctx["tests"].get("secrets") or {})]
+            if named:
+                raise Blocked(f"the scenario names the secret {', '.join(named)}; the Python library gives no run secrets yet")
+            spec = ctx["envSpec"] or {}
+            if spec.get("start") or ctx["tests"].get("stand-ins"):
+                raise Blocked("the environment starts a product or a stand-in; the Python contract library does not start one yet")
+            if sc.get("repeat"):
+                raise Blocked("the scenario has repeat; the Python contract library does not repeat a run yet")
+            # Each way in of the run is checked before the body runs, also one the body never reaches through run.driver.
+            for way in sc["through"]:
+                spec_way = (ctx["tests"].get("ways-in") or {}).get(way) or {}
+                if spec_way.get("driver") in ctx["shippedDrivers"]:
+                    raise Blocked(f'the way in "{way}" uses the kit driver "{spec_way["driver"]}", which has no Python form yet; write the test with contract.mjs')
+                guarded = [k for k in ctx["wayGuardParts"] if k in spec_way]
+                if guarded:
+                    raise Blocked(f'the way in "{way}" sets {", ".join(guarded)}, guard parts that the Python library does not apply yet; write the test with contract.mjs')
+            if not state["warmed"]:
+                warm = getattr(adapter, "warm_up", None)
+                if warm:
+                    warm(spec)
+                state["warmed"] = True
+
+        def close_drivers():
+            for drv in state["drivers"].values():
+                try:
+                    drv.close()
+                except Exception:  # the stop goes on
+                    pass
+            state["drivers"].clear()
+
+        def run_way(case, way):
+            ev = evidence()
+            ev.set_way(way, started=_now())
+            run, error = None, None
+            try:
+                with _Limit(limit_ms, limit_text):
+                    ready()
+                    run = Run(ctx, state, way, ev, adapter)
+                    state["live"] = run
+                    body(run, way)
+            except (Blocked, GateFailed) as thrown:
+                error = thrown
+            except _OutOfTime as thrown:
+                error = thrown
+            except Exception as thrown:  # a stop of the body is a fail, with its reason
+                error = thrown
+            verdict, problems = _settle(sc, way, ev, error)
+            if run is not None:
+                run.close()
+                for fn in reversed(run.cleanups):
+                    try:
+                        fn()
+                    except Exception as failed:
+                        ev.note(f"clean-up failed: {failed}")
+            if problems:
+                case.fail(state["redactor"].text("\n".join(problems)))
+
+        def not_here(case):
+            ev = evidence()
+            for a in sc["assertions"]:
+                for way in a["ways"]:
+                    ev.set_result(a["fullId"], way, "not here", {"why": f"runs-in does not list {environment}"})
+            for way in sc["ways"]:
+                ev.set_way(way, verdict="not here", started=_now(), ended=_now())
+
+        members = {"__doc__": f"{sc['id']}: {sc['title']}", "__module__": body.__module__,
+                   "tearDownClass": classmethod(lambda cls: close_drivers())}
+        if not runs_here:
+            members["test_not_here"] = lambda case: not_here(case)
+        else:
+            for way in sc["through"]:
+                members[_method_name(way)] = (lambda w: lambda case: run_way(case, w))(way)
+        return type(body.__name__, (unittest.TestCase,), members)
+
+    return wrap