bitfab 0.64.3 → 0.64.5

package/cloudReplay.py

@@ -34,6 +34,7 @@ REPLAY_COMMAND_ENV = "BITFAB_REPLAY_COMMAND"
 SDK_LANGUAGE_ENV = "BITFAB_SDK_LANGUAGE"
 CHECK_COMMAND_ENV = "BITFAB_REPLAY_CHECK"
 REQUEST_VERSION = 3
+OLDEST_DISPATCHED_VERSION = 2
 RESULT_LINE = "bitfab-replay-result "
 RESULT_CHUNK = 4000
 OUTPUT_BEGIN = "bitfab-replay-output-begin"
@@ -53,6 +54,14 @@ ITEM_ERROR_FIELDS = (
 ITEM_ID_FIELDS = ("originalTraceId", "original_trace_id", "traceId", "trace_id")
 ITEM_ERROR_LINES = 20
 ITEM_ERROR_LENGTH = 500
+SUMMARY_ERROR_ITEMS = 100
+EXPERIMENT_FIELDS = {
+    "experimentId": ("experimentId", "experiment_id"),
+    "experimentUrl": ("experimentUrl", "experiment_url"),
+    "testRunId": ("testRunId", "test_run_id"),
+    "testRunUrl": ("testRunUrl", "test_run_url"),
+    "attempts": ("attempts",),
+}
 UUID = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$")
 SHA = re.compile(r"^[0-9a-f]{40}$")
 GITHUB_REMOTE = re.compile(
@@ -60,6 +69,15 @@ GITHUB_REMOTE = re.compile(
 )
 LOG_TIMESTAMP = re.compile(r"^?\d{4}-\d\d-\d\dT[\d:.]+Z ?")
 EXPERIMENT_LINE = re.compile(rb"^\[replay\] Experiment ([0-9a-f-]{36}):")
+PROGRESS_PREFIX = b"@@bitfab:progress "
+PROGRESS_PAYLOAD_FIELDS = (
+    "input",
+    "result",
+    "originalOutput",
+    "original_output",
+    "selectiveReplay",
+    "selective_replay",
+)
 SECRET_REFERENCE = re.compile(
     r"[\"']?([A-Za-z_][A-Za-z0-9_]*)[\"']?[ \t]*:[ \t]*[\"']?\$\{\{\s*secrets(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[\s*[\"']([A-Za-z_][A-Za-z0-9_]*)[\"']\s*\])\s*\}\}"
 )
@@ -70,15 +88,21 @@ ENVIRONMENT_SETTING = re.compile(
 PUSH_TRIGGER = re.compile(
     r"^[ \t]*\"?(?:on\"?[ \t]*:.*\bpush\b|push\"?[ \t]*:)", re.MULTILINE
 )
-DECLARED_INPUTS = re.compile(r"^[ \t]*\"?execution_id\"?[ \t]*:", re.MULTILINE)
+DISPATCH_INPUTS = ("execution_id", "request")
+BRANCH_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*")
 FOLLOW_FLAGS = (
     "--cloud-status",
     "--cloud-watch",
     "--cloud-cancel",
     "--cloud-cleanup",
 )
-CLOUD_VALUE_FLAGS = (*FOLLOW_FLAGS, "--cloud-request-id", "--cloud-timeout")
-CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach")
+CLOUD_VALUE_FLAGS = (
+    *FOLLOW_FLAGS,
+    "--cloud-request-id",
+    "--cloud-timeout",
+    "--cloud-ref",
+)
+CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach", "--cloud-full-output")
 RENAMED_FLAGS = {
     "--cloud-dry-run": "--cloud-preview",
     "--cloud-check": "--dry-run",
@@ -114,11 +138,36 @@ does not ignore) from the same directory, and this command prints the replay's o
 and exits with its exit code. Credential-like files are refused. Requires git and
 Python 3.10+ on macOS or Linux, with a github.com origin.
 
+The replay's stderr comes back as it printed it. Its stdout result comes back as a
+short summary instead of every item: the experiment ID and URL, how many traces were
+replayed, same, changed, and errored, each errored trace with its error, and the exit
+code. Every item's input and output are already saved in the experiment in Bitfab.
+
 Options added by --cloud:
   --cloud-preview        List what the snapshot would contain and push nothing.
   --cloud-detach         Return after dispatch instead of waiting.
+  --cloud-full-output    Print the replay's whole stdout result, every item included,
+                         instead of the summary. Large runs can exceed what the
+                         GitHub job log holds.
   --cloud-timeout MIN    Stop the replay after MIN minutes (1..7200).
   --cloud-request-id ID  Recover a submission whose response was lost.
+  --cloud-ref BRANCH     Replay BRANCH as it is on GitHub, with no snapshot of your
+                         working tree, and record the experiment on BRANCH. Meant for
+                         CI jobs, such as a nightly replay of dev. Needs only GitHub
+                         access that can run Actions, and a workflow on BRANCH that
+                         declares a request input.
+
+A workflow can also dispatch bitfab-replay.yml itself on any branch other than a
+bitfab-replay/ snapshot branch, passing the replay as JSON in the request input:
+
+  {"args": ["--registry", "scripts/replay.ts", "classify", "--dataset-ids", "UUID"],
+   "cwd": "web", "timeoutMinutes": 60}
+
+args are the replay options without --cloud. cwd is the directory to replay from,
+relative to the repository root, and defaults to the root. timeoutMinutes and
+fullOutput (true) match --cloud-timeout and --cloud-full-output. version and id are
+optional: version names the request format (3 now, 2 still accepted), and id is the
+execution UUID, which must match an execution_id input when the workflow has one.
 
 Follow a replay by the execution UUID it prints:
   --cloud-watch ID | --cloud-status ID | --cloud-cancel ID | --cloud-cleanup ID
@@ -586,6 +635,16 @@ def parse(argv):
             timeout.isdigit() and 1 <= int(timeout) <= 7200
         ):
             raise ValueError("--cloud-timeout must be 1..7200 minutes")
+        branch = cloud.get("--cloud-ref")
+        if branch is not None and (
+            not BRANCH_NAME.fullmatch(branch)
+            or branch.startswith(PREFIX)
+            or ".." in branch
+            or branch.endswith((".", "/", ".lock"))
+        ):
+            raise ValueError(
+                f"--cloud-ref takes a branch name such as dev, other than a {PREFIX} snapshot branch"
+            )
         operation = "submit"
         execution_id = cloud.get("--cloud-request-id") or str(uuid.uuid4())
     if not UUID.fullmatch(execution_id):
@@ -648,6 +707,8 @@ def replay_request(root, parsed):
     }
     if "--cloud-timeout" in parsed["cloud"]:
         request["timeoutMinutes"] = int(parsed["cloud"]["--cloud-timeout"])
+    if "--cloud-full-output" in parsed["switches"]:
+        request["fullOutput"] = True
     return request, files
 
 
@@ -770,6 +831,8 @@ def execution_lock(directory, execution_id):
 
 
 def find_run(record):
+    if record.get("snapshot") is False:
+        return find_branch_run(record)
     query = urlencode(
         {"event": "workflow_dispatch", "branch": record["branch"], "per_page": 100}
     )
@@ -788,6 +851,31 @@ def find_run(record):
     return matches[0] if matches else None
 
 
+def find_branch_run(record):
+    query = urlencode(
+        {
+            "event": "workflow_dispatch",
+            "branch": record["branch"],
+            "created": ">=" + record["dispatchedAt"],
+            "per_page": 100,
+        }
+    )
+    runs = api(
+        record["repository"], f"actions/workflows/{record['workflow']}/runs?{query}"
+    )
+    matches = [
+        run
+        for run in runs["workflow_runs"]
+        if run["head_branch"] == record["branch"]
+        and run.get("created_at", "") >= record["dispatchedAt"]
+    ]
+    if len(matches) > 1:
+        raise ValueError(
+            f"Several runs were dispatched on {record['branch']} at about the same time, so this execution cannot tell which is its own; find it in the Actions tab"
+        )
+    return matches[0] if matches else None
+
+
 def status(record, *, fetch_result=True):
     if record["state"] == "prepared":
         return record
@@ -799,6 +887,8 @@ def status(record, *, fetch_result=True):
     if run is None:
         record["state"] = "dispatch_unknown"
         return record
+    if record.get("snapshot") is False and run["head_branch"] == record["branch"]:
+        record["sha"] = run["head_sha"]
     if run["head_sha"] != record["sha"] or run["head_branch"] != record["branch"]:
         raise ValueError("GitHub run does not match the recorded execution")
     record.update(
@@ -871,9 +961,7 @@ def outcome(record):
         raise ValueError("Replay result does not match this execution")
     record["exitCode"] = result["exitCode"]
     parsed = last_json(result.get("stdout", "")) or {}
-    test_run = result.get("testRunId") or parsed.get(
-        "testRunId", parsed.get("test_run_id")
-    )
+    test_run = result.get("testRunId") or experiment_fields(parsed).get("testRunId")
     if isinstance(test_run, str) and UUID.fullmatch(test_run):
         record["testRunId"] = test_run
     if result.get("stoppedEarly"):
@@ -903,6 +991,17 @@ def print_outcome(found):
             f"The replay also printed {result['stdoutOmittedBytes'] // (1024 * 1024)} MiB to stdout before its result, such as a library's logging; only the end was kept"
         )
     print(result.get("stdout", ""), end="", flush=True)
+    summary = last_json(result.get("stdout", "")) or {}
+    if summary.get("cloudSummary") is True:
+        progress(
+            "Printed a summary of the result; every item's input and output are in the experiment"
+            + (
+                f" at {summary['experimentUrl']}"
+                if summary.get("experimentUrl")
+                else ""
+            )
+            + ". Add --cloud-full-output to print every item here instead"
+        )
     if record.get("stoppedEarly"):
         progress(
             f"Replay stopped early: {record['stoppedEarly']}. Traces that finished are saved"
@@ -919,6 +1018,9 @@ def print_outcome(found):
 def cleanup(root, record):
     if record.get("cleaned"):
         return
+    if record.get("snapshot") is False:
+        record["cleaned"] = True
+        return
     if record.get("state") not in ("completed", "prepared"):
         raise ValueError(
             "Cleanup requires a confirmed completed GitHub run; cancel and wait first"
@@ -947,7 +1049,7 @@ def http_status(error):
     return f" (HTTP {status})" if status else ""
 
 
-def preflight(repo, workflow):
+def preflight(repo, workflow, *, pushes=True):
     access = github_access()
     try:
         details = api(repo, "")
@@ -955,7 +1057,7 @@ def preflight(repo, workflow):
         raise ValueError(
             f"GitHub access from {access['source']} cannot read {repo}{http_status(error)}; use an account or token with write access to it"
         ) from error
-    if (details or {}).get("permissions", {}).get("push") is False:
+    if pushes and (details or {}).get("permissions", {}).get("push") is False:
         raise ValueError(
             f"GitHub access from {access['source']} cannot push to {repo}; use an account or token with write access to it"
         )
@@ -1036,21 +1138,43 @@ def submit(root, repo, workflow, parsed, path):
         ) from error
     record["state"] = "dispatch_unknown"
     save(path, record)
-    payload = {"ref": record["branch"]}
-    if DECLARED_INPUTS.search(
+    declared = declared_inputs(
         within(root, f"{WORKFLOW_DIRECTORY}/{workflow}").read_text()
-    ):
-        payload["inputs"] = {"execution_id": execution_id, "request": "-"}
+    )
+    return dispatch(record, {"execution_id": execution_id, "request": "-"}, declared)
+
+
+def declared_inputs(text):
+    return {
+        name
+        for name in DISPATCH_INPUTS
+        if re.search(rf"^[ \t]*\"?{name}\"?[ \t]*:", text, re.MULTILINE)
+    }
+
+
+def dispatch(record, inputs, declared, *, run_details=False):
+    payload = {"ref": record["branch"]}
+    sent = {name: value for name, value in inputs.items() if name in declared}
+    if sent:
+        payload["inputs"] = sent
+    path = f"actions/workflows/{record['workflow']}/dispatches"
     try:
-        response = api(
-            repo,
-            f"actions/workflows/{workflow}/dispatches",
-            method="POST",
-            payload=payload,
-        )
+        try:
+            response = api(
+                record["repository"],
+                path,
+                method="POST",
+                payload={**payload, "return_run_details": True}
+                if run_details
+                else payload,
+            )
+        except RuntimeError as error:
+            if not run_details or getattr(error, "http_status", None) != 422:
+                raise
+            response = api(record["repository"], path, method="POST", payload=payload)
     except RuntimeError as error:
         raise ValueError(
-            f"Dispatching {workflow} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-watch {execution_id}"
+            f"Dispatching {record['workflow']} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-watch {record['id']}"
         ) from error
     if isinstance(response, dict) and response.get("workflow_run_id"):
         record["runId"] = response["workflow_run_id"]
@@ -1059,6 +1183,84 @@ def submit(root, repo, workflow, parsed, path):
     return record
 
 
+def branch_head(repo, branch):
+    try:
+        found = api(repo, f"git/ref/heads/{branch}")
+    except RuntimeError as error:
+        if getattr(error, "http_status", None) == 404:
+            raise ValueError(f"{repo} has no branch {branch}") from error
+        raise
+    sha = ((found or {}).get("object") or {}).get("sha", "")
+    if not SHA.fullmatch(sha):
+        raise ValueError(f"GitHub gave no commit for branch {branch}")
+    return sha
+
+
+def branch_workflow(repo, workflow, sha):
+    path = f"{WORKFLOW_DIRECTORY}/{workflow}"
+    try:
+        found = api(repo, f"contents/{path}?ref={sha}")
+    except RuntimeError as error:
+        raise ValueError(
+            f"Could not read {path} at {sha[:12]}{http_status(error)}; the branch needs the workflow"
+        ) from error
+    return base64.b64decode((found or {}).get("content", "")).decode(errors="replace")
+
+
+def submit_as_is(root, repo, workflow, parsed, path):
+    execution_id = parsed["id"]
+    branch = parsed["cloud"]["--cloud-ref"]
+    request, _ = replay_request(root, parsed)
+    sha = branch_head(repo, branch)
+    if "--cloud-preview" in parsed["switches"]:
+        return {
+            "preview": True,
+            "repository": repo,
+            "branch": branch,
+            "sha": sha,
+            "request": request,
+        }
+    if path.exists():
+        record = json.loads(path.read_text())
+        if (
+            record["request"] != request
+            or record["repository"].lower() != repo.lower()
+            or record["branch"] != branch
+        ):
+            raise ValueError("Execution ID already belongs to a different request")
+        status(record, fetch_result=False)
+        return record
+    preflight(repo, workflow, pushes=False)
+    declared = declared_inputs(branch_workflow(repo, workflow, sha))
+    if "request" not in declared:
+        raise ValueError(
+            f"{WORKFLOW_DIRECTORY}/{workflow} on {branch} declares no request input, so it cannot take a replay; add request (type: string, required: false) under on.workflow_dispatch.inputs and land it on {branch}"
+        )
+    record = {
+        "id": execution_id,
+        "repository": repo,
+        "workflow": workflow,
+        "branch": branch,
+        "snapshot": False,
+        "request": request,
+        "state": "dispatch_unknown",
+        "sha": sha,
+        "dispatchedAt": time.strftime(
+            "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 5)
+        ),
+    }
+    save(path, record)
+    progress(
+        f"Replaying {branch} at {sha[:12]} as it is on GitHub, without local changes. Execution {execution_id}; resume with --cloud-watch {execution_id}"
+    )
+    return dispatch(
+        record,
+        {"execution_id": execution_id, "request": json.dumps(request)},
+        declared,
+        run_details=True,
+    )
+
+
 def report_steps(record, shown):
     job = replay_job(record)
     for step in (job or {}).get("steps", []):
@@ -1104,7 +1306,8 @@ def run_cli(argv):
     path = directory / f"{execution_id}.json"
     with execution_lock(directory, execution_id):
         if operation == "submit":
-            record = submit(root, repo, workflow, parsed, path)
+            submitter = submit_as_is if "--cloud-ref" in parsed["cloud"] else submit
+            record = submitter(root, repo, workflow, parsed, path)
             if record.get("preview"):
                 return record, None
         else:
@@ -1180,14 +1383,86 @@ def snapshot_request(root, commit):
     return request
 
 
+def dispatched_inputs():
+    path = os.environ.get("GITHUB_EVENT_PATH")
+    if not path or not Path(path).is_file():
+        return {}
+    inputs = json.loads(Path(path).read_text()).get("inputs")
+    return inputs if isinstance(inputs, dict) else {}
+
+
+def decode_request(text):
+    try:
+        return json.loads(text)
+    except ValueError:
+        pass
+    try:
+        return json.loads(base64.b64decode(text, validate=True))
+    except ValueError as error:
+        raise ValueError(
+            "The request input is neither JSON nor base64-encoded JSON"
+        ) from error
+
+
+def dispatched_request():
+    branch = os.environ.get("GITHUB_REF_NAME", "")
+    inputs = dispatched_inputs()
+    text = str(inputs.get("request") or "").strip()
+    if text in ("", "-"):
+        raise ValueError(
+            f"A replay of {branch} takes its request from the workflow_dispatch request input, and this run has none; start replays with bitfab-replay --cloud, or --cloud --cloud-ref {branch} to replay the branch as it is"
+        )
+    request = decode_request(text)
+    if not isinstance(request, dict):
+        raise ValueError("The request input must be a JSON object")
+    version = request.get("version", REQUEST_VERSION)
+    if not isinstance(version, int) or version < OLDEST_DISPATCHED_VERSION:
+        raise ValueError(
+            f"Request version {version!r} is not supported; send version {REQUEST_VERSION}, or leave version out"
+        )
+    if version > REQUEST_VERSION:
+        raise ValueError(
+            f"This replay was submitted by a newer SDK than the one {branch} installs; the request is version {version} and this SDK reads up to {REQUEST_VERSION}"
+        )
+    execution_id = request.get("id") or inputs.get("execution_id") or str(uuid.uuid4())
+    if not isinstance(execution_id, str) or not UUID.fullmatch(execution_id):
+        raise ValueError("The request id must be a UUID")
+    if inputs.get("execution_id") and inputs["execution_id"] != execution_id:
+        raise ValueError("The request id does not match the execution_id input")
+    args = request.get("args")
+    validate_arguments(args)
+    if request.get("check") is True and "--dry-run" not in args:
+        args = [*args, "--dry-run"]
+    normalized = {
+        "version": REQUEST_VERSION,
+        "id": execution_id,
+        "cwd": request.get("cwd") or ".",
+        "args": args,
+    }
+    timeout = request.get("timeoutMinutes")
+    if timeout is not None:
+        if not isinstance(timeout, int) or not 1 <= timeout <= 7200:
+            raise ValueError("timeoutMinutes must be 1..7200")
+        normalized["timeoutMinutes"] = timeout
+    if request.get("fullOutput") is True:
+        normalized["fullOutput"] = True
+    return normalized
+
+
+def runner_request(root, commit):
+    if os.environ.get("GITHUB_REF_NAME", "").startswith(PREFIX):
+        return snapshot_request(root, commit)
+    return dispatched_request()
+
+
 def execute():
     if os.environ.get("GITHUB_RUN_ATTEMPT") != "1":
         raise ValueError("Submit a new replay instead of rerunning an Actions job")
     root = root_directory()
     commit = os.environ["GITHUB_SHA"]
     if git(root, "rev-parse", "HEAD") != commit:
-        raise ValueError("Runner checkout does not match the dispatched snapshot SHA")
-    request = snapshot_request(root, commit)
+        raise ValueError("Runner checkout does not match the dispatched commit")
+    request = runner_request(root, commit)
     directory = within(root, request["cwd"])
     check_secrets(root)
     names = {option_name(value) for value in request["args"]}
@@ -1203,7 +1478,11 @@ def execute():
     print(OUTPUT_BEGIN, flush=True)
     try:
         code, stdout, omitted, stopped = run_command(
-            directory, args, request.get("timeoutMinutes"), experiment
+            directory,
+            args,
+            request.get("timeoutMinutes"),
+            experiment,
+            summarize=request.get("fullOutput") is not True,
         )
     finally:
         print(OUTPUT_END, flush=True)
@@ -1286,10 +1565,75 @@ def item_errors(items):
         text = " ".join(str(error).split())
         if len(text) > ITEM_ERROR_LENGTH:
             text = text[:ITEM_ERROR_LENGTH] + "..."
-        errors.append((trace, text))
+        errored = {"originalTraceId": trace, "error": text}
+        if isinstance(item.get("attempt"), int):
+            errored["attempt"] = item["attempt"]
+        errors.append(errored)
     return errors
 
 
+def item_field(item, camel, snake):
+    return item[camel] if camel in item else item.get(snake)
+
+
+def canonical(value):
+    return json.dumps(value, separators=(",", ":"))
+
+
+def experiment_fields(parsed):
+    fields = {}
+    for name, spellings in EXPERIMENT_FIELDS.items():
+        value = next(
+            (parsed[key] for key in spellings if parsed.get(key) is not None), None
+        )
+        if value is not None:
+            fields[name] = value
+    return fields
+
+
+def replay_summary(parsed):
+    items = [item for item in parsed["items"] if isinstance(item, dict)]
+    replayed = [item for item in items if not carried_over(item)]
+    errors = item_errors(replayed)
+    counts = {
+        "replayed": len(replayed),
+        "carriedOver": len(items) - len(replayed),
+        "same": 0,
+        "changed": 0,
+        "matchedExpected": 0,
+        "missedExpected": 0,
+        "errored": len(errors),
+    }
+    for item in replayed:
+        if item_errored(item):
+            continue
+        equal = canonical(item.get("result")) == canonical(
+            item_field(item, "originalOutput", "original_output")
+        )
+        if item_field(item, "ingestionType", "ingestion_type") == "seeded":
+            counts["matchedExpected" if equal else "missedExpected"] += 1
+        else:
+            counts["same" if equal else "changed"] += 1
+    summary = {
+        "cloudSummary": True,
+        **experiment_fields(parsed),
+        "counts": counts,
+        "erroredItems": errors[:SUMMARY_ERROR_ITEMS],
+    }
+    if len(errors) > SUMMARY_ERROR_ITEMS:
+        summary["erroredItemsOmitted"] = len(errors) - SUMMARY_ERROR_ITEMS
+    return summary
+
+
+def result_summary(stdout):
+    parsed = last_json(stdout) or {}
+    if parsed.get("cloudSummary") is True:
+        return parsed
+    if isinstance(parsed.get("items"), list):
+        return replay_summary(parsed)
+    return None
+
+
 def item_errored(item):
     return isinstance(item, dict) and any(
         item.get(field) is not None for field in ITEM_ERROR_FIELDS
@@ -1306,8 +1650,24 @@ def raise_interrupt(signum, frame):
     raise KeyboardInterrupt
 
 
-def forward_stderr(stream, experiment):
+def without_item_payload(line):
+    try:
+        event = json.loads(line[len(PROGRESS_PREFIX) :])
+    except ValueError:
+        return line
+    item = event.get("item") if isinstance(event, dict) else None
+    if not isinstance(item, dict):
+        return line
+    event["item"] = {
+        key: value for key, value in item.items() if key not in PROGRESS_PAYLOAD_FIELDS
+    }
+    return PROGRESS_PREFIX + json.dumps(event).encode() + b"\n"
+
+
+def forward_stderr(stream, experiment, summarize):
     for line in iter(stream.readline, b""):
+        if summarize and line.startswith(PROGRESS_PREFIX):
+            line = without_item_payload(line)
         sys.stdout.write(line.decode(errors="replace"))
         sys.stdout.flush()
         if "id" not in experiment:
@@ -1333,6 +1693,15 @@ def last_json(text):
     return trailing_json(text)[1]
 
 
+def relayed_stdout(output, code, summarize):
+    if summarize:
+        output.seek(0)
+        parsed = last_json(output.read().decode(errors="replace"))
+        if isinstance(parsed, dict) and isinstance(parsed.get("items"), list):
+            return json.dumps({**replay_summary(parsed), "exitCode": code}) + "\n", 0
+    return carried_stdout(output)
+
+
 def carried_stdout(output):
     start = max(0, os.fstat(output.fileno()).st_size - OUTPUT_LIMIT)
     output.seek(start)
@@ -1365,34 +1734,34 @@ def annotate(result):
 
 
 def summary_markdown(result):
-    parsed = last_json(result["stdout"]) or {}
-    test_run = result.get("testRunId") or parsed.get(
-        "testRunId", parsed.get("test_run_id")
-    )
-    items = parsed.get("items") if isinstance(parsed.get("items"), list) else []
-    replayed = [item for item in items if not carried_over(item)]
-    errors = item_errors(replayed)
+    summary = result_summary(result["stdout"]) or {}
+    counts = summary.get("counts", {})
+    test_run = result.get("testRunId") or summary.get("testRunId")
+    errors = summary.get("erroredItems", [])
     lines = [f"Commit: `{result['commitSha']}`", f"Exit code: {result['exitCode']}"]
     if test_run:
         lines.insert(0, f"Test run: `{test_run}`")
     if result.get("stoppedEarly"):
         lines.append(f"Stopped early: {result['stoppedEarly']}")
-    elif items:
-        lines.append(f"Replayed: {len(replayed)}, errored: {len(errors)}")
+    elif counts.get("replayed") or counts.get("carriedOver"):
+        lines.append(
+            f"Replayed: {counts['replayed']}, same: {counts['same']}, changed: {counts['changed']}, errored: {counts['errored']}"
+        )
     text = "### Bitfab replay\n\n" + "\n\n".join(lines) + "\n"
     if errors:
         shown = [
-            f"trace {trace}: {error}" for trace, error in errors[:ITEM_ERROR_LINES]
+            f"trace {error['originalTraceId']}: {error['error']}"
+            for error in errors[:ITEM_ERROR_LINES]
         ]
-        if len(errors) > len(shown):
-            shown.append(f"and {len(errors) - len(shown)} more errored items")
+        if counts.get("errored", 0) > len(shown):
+            shown.append(f"and {counts['errored'] - len(shown)} more errored items")
         text += "\n#### Errored items\n\n" + "".join(
             f"- `{line.replace('`', chr(39))}`\n" for line in shown
         )
     return text
 
 
-def run_command(directory, args, timeout, experiment):
+def run_command(directory, args, timeout, experiment, *, summarize=True):
     stopped = None
     with tempfile.TemporaryFile() as output:
         with subprocess.Popen(
@@ -1404,7 +1773,9 @@ def run_command(directory, args, timeout, experiment):
             start_new_session=True,
         ) as child:
             reader = threading.Thread(
-                target=forward_stderr, args=(child.stderr, experiment), daemon=True
+                target=forward_stderr,
+                args=(child.stderr, experiment, summarize),
+                daemon=True,
             )
             reader.start()
             try:
@@ -1431,9 +1802,9 @@ def run_command(directory, args, timeout, experiment):
                 raise
             finally:
                 reader.join(timeout=5)
-            code = child.returncode
-        text, omitted = carried_stdout(output)
-    return (code if code >= 0 else 1), text, omitted, stopped
+            code = child.returncode if child.returncode >= 0 else 1
+        text, omitted = relayed_stdout(output, code, summarize)
+    return code, text, omitted, stopped
 
 
 def stop(child):
@@ -1696,7 +2067,11 @@ def workflow_document(steps, run, directory, env, *, runs_on, environment):
     ]
     return {
         "name": "Bitfab replay",
-        "on": "workflow_dispatch",
+        "on": {
+            "workflow_dispatch": {
+                "inputs": {"request": {"type": "string", "required": False}}
+            }
+        },
         "permissions": {"contents": "read"},
         "jobs": {"replay": job},
     }