bitfab 0.62.3 → 0.62.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ea9a0439252adacbe50f5c2bc210b5a9708eb772b9998c3660008b10e14dd95
-  data.tar.gz: 0cd4bfab124dac3a35d3653f2fbef02c6b7e6671bbab36fa9f3b583652ed8339
+  metadata.gz: f7055ee2b869bb3acd9740221ebe869d67564ec79faa41ec34c3de2d1e5c309c
+  data.tar.gz: 1e1fa48e73ebe238b50a1d88804512026dfa0e1faa833422c1646afce461d25c
 SHA512:
-  metadata.gz: 4e191d599d446e776599c9b01014768e95710f23e9ca27f2c8618008eb3b454718d629f2533d242b412520e6c309a5a390ff65e66d1bd10a23a865b6c0d53654
-  data.tar.gz: 74f6a2fa1301b226afca67d30490b5ed0095a28cae9aa8513d89291a7b06e7f95303028fe104ff672a79e57efcdc3c8bfd491e643a1ec7fe33f5a478936160d7
+  metadata.gz: 79492cc60cb6ef608357d1bde581d632fdbd8140cf89b1a622e4a5b9efbbd265ac558d0342efa0d165252ab43233aa458bd0eb2a2f3e59770c02244742889eea
+  data.tar.gz: 213a1c8ce50982a81b59fdfce1ecc6e1f724e6e75833a08793da16307037a2c4e5b10d631365c2b76468b0596bfcc524426bd986a39539a9339357b58ef7ee06

data/lib/bitfab/client.rb

@@ -37,7 +37,7 @@ module Bitfab
 
     attr_reader :service_url
 
-    attr_reader :datasets, :assertion_categories, :organization_members, :traces, :labels, :graders
+    attr_reader :datasets, :assertion_categories, :organization_members, :traces, :labels, :graders, :experiments
 
     def initialize(api_key: nil, service_url: nil, capture_enabled: nil, enabled: nil, strict: false, simulation_plan: true, timeout: 120, trace_across_threads: nil, env_vars: nil, baml_client: nil, db_snapshot: nil)
       @db_snapshot = DbSnapshot.normalize_config(db_snapshot)
@@ -59,6 +59,7 @@ module Bitfab
       @traces = Traces.new(@http_client)
       @labels = Labels.new(@http_client)
       @graders = Graders.new(@http_client)
+      @experiments = Experiments.new(@http_client)
       # Mock overrides registered via register_mock_override, applied to every
       # replay on this client (after any per-call mock_override). Instance
       # state, no global; clear_mock_overrides resets it.
@@ -197,7 +198,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -209,6 +210,7 @@ module Bitfab
         trace_ids:,
         name:,
         notes:,
+        metadata:,
         concurrency:,
         max_concurrency:,
         attempts:,

data/lib/bitfab/cloudReplay.py

@@ -11,6 +11,7 @@ import signal
 import subprocess
 import sys
 import tempfile
+import threading
 import time
 import uuid
 from pathlib import Path
@@ -20,6 +21,20 @@ CONFIG = ".bitfab/cloud.json"
 PREFIX = "bitfab-replay/"
 DEFAULT_SECRET_PREFIX = "BITFAB_CLOUD_"
 API_VERSION = "2026-03-10"
+# The SDK finds runs by this title and feeds them through these inputs. Never change them.
+RUN_NAME = "Bitfab replay ${{ inputs.execution_id }}"
+RUNNER_ENV = {
+    "BITFAB_CLOUD_REQUEST": "${{ inputs.request }}",
+    "BITFAB_EXECUTION_ID": "${{ inputs.execution_id }}",
+    "BITFAB_COMMIT_SHA": "${{ github.sha }}",
+}
+# The runner reports its result as a job annotation with this title, so the workflow
+# needs no upload step. Older workflows still upload RESULT_FILE as an artifact.
+RESULT_TITLE = "Bitfab replay result"
+RESULT_FILE = "bitfab-cloud-result.json"
+# Steps older setups generated that the SDK now owns.
+OLD_UPLOAD_STEP = "Save replay identity"
+OLD_JOB_TIMEOUT = 35
 ITEM_ERROR_FIELDS = (
     "error",
     "traceError",
@@ -30,13 +45,17 @@ ITEM_ERROR_FIELDS = (
 UUID = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$")
 SHA = re.compile(r"^[0-9a-f]{40}$")
 PIPELINE = re.compile(r"^[\w.][\w.-]*$")
+# Every SDK's replay prints this to stderr as soon as the server creates the experiment.
+EXPERIMENT_LINE = re.compile(rb"^\[replay\] Experiment ([0-9a-f-]{36}):")
 HELP = """GitHub cloud replay (requires git, gh login, and Python 3.10+).
   --cloud PIPELINE --trace-ids UUID[,UUID] [--registry PATH]
     [--max-concurrency 1..32] [--cloud-request-id UUID]
     [--cloud-include PATH ...] [--cloud-dry-run] [--cloud-detach]
+    [--cloud-timeout MINUTES]
   --cloud-status UUID | --cloud-watch UUID | --cloud-cancel UUID
   --cloud-cleanup UUID
-  --cloud-init --config SPEC | --cloud-secrets --env-file FILE [NAME ...]
+  --cloud-init [--config SPEC]   (creates a setup, or brings an existing one up to date)
+  --cloud-secrets --env-file FILE [NAME ...]
 PIPELINE may be any pipeline in the registry, unless .bitfab/cloud.json names one.
 Snapshot tracked working files without changing HEAD, the index, or local files.
 New files require explicit --cloud-include. Credentials and ignored files are refused.
@@ -266,6 +285,8 @@ def validate_config(config):
         raise ValueError(
             "The runner command must execute locally, not recursively dispatch"
         )
+    if "cliCommand" in config:
+        validate_cli_command(config["cliCommand"])
     if config.get("pushTriggersReviewed") is not True:
         raise ValueError(
             "Setup must review push-triggered CI/deployments and set pushTriggersReviewed=true"
@@ -285,6 +306,18 @@ def validate_config(config):
         raise ValueError("secrets must be uppercase environment variable names")
 
 
+def validate_cli_command(cli_command):
+    if (
+        not isinstance(cli_command, list)
+        or not cli_command
+        or not all(isinstance(v, str) and v and "\x00" not in v for v in cli_command)
+        or any(v.startswith("--cloud") for v in cli_command)
+    ):
+        raise ValueError(
+            "cliCommand must be the argument array that starts the SDK's bitfab-replay command from workingDirectory"
+        )
+
+
 def check_pipeline(config, pipeline):
     if not isinstance(pipeline, str) or not PIPELINE.fullmatch(pipeline):
         raise ValueError("Invalid pipeline name")
@@ -342,6 +375,7 @@ def parse(argv):
     parser.add_argument("--cloud-include", action="append", default=[])
     parser.add_argument("--cloud-dry-run", action="store_true")
     parser.add_argument("--cloud-detach", action="store_true")
+    parser.add_argument("--cloud-timeout", type=int)
     args = parser.parse_args(argv)
     operations = [
         name
@@ -363,6 +397,9 @@ def parse(argv):
             raise ValueError("Supply 1..100 explicit trace UUIDs")
         if not 1 <= args.max_concurrency <= 32:
             raise ValueError("--max-concurrency must be 1..32")
+        # Self-hosted runners allow jobs of up to 5 days; GitHub-hosted ones stop at 6 hours.
+        if args.cloud_timeout is not None and not 1 <= args.cloud_timeout <= 7200:
+            raise ValueError("--cloud-timeout must be 1..7200 minutes")
     if not UUID.fullmatch(execution_id):
         raise ValueError("Execution ID must be a UUID")
     return args, operation, execution_id
@@ -547,39 +584,82 @@ def status(record, *, fetch_result=True):
     if (
         fetch_result
         and record["state"] == "completed"
-        and record["conclusion"] == "success"
         and not record.get("testRunId")
+        and not record.get("resultChecked")
     ):
-        with tempfile.TemporaryDirectory(prefix="bitfab-cloud-result-") as directory:
-            command(
-                [
-                    "gh",
-                    "run",
-                    "download",
-                    str(record["runId"]),
-                    "--repo",
-                    record["repository"],
-                    "--name",
-                    "bitfab-replay-" + record["id"],
-                    "--dir",
-                    directory,
-                ]
-            )
-            artifact = Path(directory) / "bitfab-cloud-result.json"
-            if artifact.is_symlink() or artifact.stat().st_size > 4096:
-                raise ValueError("Invalid replay result artifact")
-            result = json.loads(artifact.read_text())
+        success = record["conclusion"] == "success"
+        try:
+            result = read_result(record)
+        except RuntimeError:
+            # A run that failed before its replay started reports no result.
+            if success:
+                raise
+            record["resultChecked"] = True
+            return record
+        if (
+            result.get("executionId") != record["id"]
+            or result.get("commitSha") != record["sha"]
+            or not UUID.fullmatch(result.get("testRunId", ""))
+        ):
+            raise ValueError("Replay result does not match this execution")
+        counts = replay_counts(result)
+        record["testRunId"] = result["testRunId"]
+        if isinstance(result.get("stoppedEarly"), str):
+            record["stoppedEarly"] = result["stoppedEarly"]
+        if counts is not None:
+            record["replayed"], record["errored"] = counts
+    return record
+
+
+def read_result(record):
+    try:
+        result = read_result_annotation(record)
+    except RuntimeError:
+        # Older workflows still upload the result, so a failed Checks API read
+        # must not hide it.
+        result = None
+    return result or read_result_artifact(record)
+
+
+def read_result_annotation(record):
+    repo = record["repository"]
+    jobs = api(repo, f"actions/runs/{record['runId']}/jobs?per_page=100") or {}
+    for job in jobs.get("jobs", []):
+        check_run = str(job.get("check_run_url", "")).rsplit("/", 1)[-1]
+        if not check_run.isdigit():
+            continue
+        for note in api(repo, f"check-runs/{check_run}/annotations?per_page=100") or []:
             if (
-                result.get("executionId") != record["id"]
-                or result.get("commitSha") != record["sha"]
-                or not UUID.fullmatch(result.get("testRunId", ""))
+                note.get("title") == RESULT_TITLE
+                and len(note.get("message", "")) <= 4096
             ):
-                raise ValueError("Replay result artifact does not match this execution")
-            counts = replay_counts(result)
-            record["testRunId"] = result["testRunId"]
-            if counts is not None:
-                record["replayed"], record["errored"] = counts
-    return record
+                try:
+                    return json.loads(base64.b64decode(note["message"], validate=True))
+                except ValueError as error:
+                    raise ValueError("Invalid replay result annotation") from error
+    return None
+
+
+def read_result_artifact(record):
+    with tempfile.TemporaryDirectory(prefix="bitfab-cloud-result-") as directory:
+        command(
+            [
+                "gh",
+                "run",
+                "download",
+                str(record["runId"]),
+                "--repo",
+                record["repository"],
+                "--name",
+                "bitfab-replay-" + record["id"],
+                "--dir",
+                directory,
+            ]
+        )
+        artifact = Path(directory) / RESULT_FILE
+        if artifact.is_symlink() or artifact.stat().st_size > 4096:
+            raise ValueError("Invalid replay result artifact")
+        return json.loads(artifact.read_text())
 
 
 def cleanup(root, record):
@@ -643,6 +723,8 @@ def run_cli(argv):
                 "traceIds": args.trace_ids.split(","),
                 "maxConcurrency": args.max_concurrency,
             }
+            if args.cloud_timeout is not None:
+                request["timeoutMinutes"] = args.cloud_timeout
             if args.cloud_dry_run:
                 return {
                     "dryRun": True,
@@ -729,17 +811,24 @@ def run_cli(argv):
             save(path, record)
             return record
         if operation == "watch" or (operation == "submit" and not args.cloud_detach):
-            deadline = time.monotonic() + 40 * 60
-            while record["state"] != "completed" and time.monotonic() < deadline:
+            # Once GitHub shows the run, its job timeout bounds the wait and Ctrl-C detaches.
+            # A dispatch GitHub never shows as a run would otherwise be waited on forever.
+            started = time.monotonic()
+            while record["state"] != "completed":
                 status(record)
                 save(path, record)
+                if (
+                    record["state"] == "dispatch_unknown"
+                    and time.monotonic() - started >= 10 * 60
+                ):
+                    raise ValueError(
+                        "GitHub has shown no run for this dispatch after 10 minutes. Check Actions, then resume with --cloud-watch "
+                        + execution_id
+                        + " or remove the snapshot branch with --cloud-cleanup "
+                        + execution_id
+                    )
                 if record["state"] != "completed":
                     time.sleep(5)
-            if record["state"] != "completed":
-                raise ValueError(
-                    "Watch timed out; the job may still run. Resume with --cloud-watch "
-                    + execution_id
-                )
         if record["state"] == "completed" or (
             operation == "cleanup" and record["state"] == "prepared"
         ):
@@ -763,6 +852,7 @@ def execute():
     check_pipeline(config, request["pipeline"])
     if request["id"] != os.environ["BITFAB_EXECUTION_ID"]:
         raise ValueError("Replay request does not match the configured execution")
+    timeout = request.get("timeoutMinutes")
     parse(
         [
             "--cloud",
@@ -773,6 +863,7 @@ def execute():
             str(request["maxConcurrency"]),
             "--cloud-request-id",
             request["id"],
+            *([] if timeout is None else ["--cloud-timeout", str(timeout)]),
         ]
     )
     if not os.environ.get("BITFAB_API_KEY"):
@@ -786,32 +877,100 @@ def execute():
         str(request["maxConcurrency"]),
         "--no-code-change",
     ]
+    experiment = {}
+    # GitHub cancels and times out a job with SIGINT then SIGTERM; turn SIGTERM into the
+    # same interrupt so the replay is stopped cleanly and its experiment is still reported.
+    previous = signal.signal(signal.SIGTERM, raise_interrupt)
+    try:
+        summary = run_replay(root, config, request, args, timeout, experiment)
+    except BaseException as error:
+        if UUID.fullmatch(experiment.get("id", "")):
+            write_result(
+                {
+                    "executionId": request["id"],
+                    "commitSha": os.environ["GITHUB_SHA"],
+                    "testRunId": experiment["id"],
+                    "stoppedEarly": (str(error) or type(error).__name__)[:300],
+                }
+            )
+        raise
+    finally:
+        signal.signal(signal.SIGTERM, previous)
+    write_result(summary)
+    return summary
+
+
+def raise_interrupt(signum, frame):
+    raise KeyboardInterrupt
+
+
+def forward_stderr(stream, experiment):
+    for line in iter(stream.readline, b""):
+        sys.stderr.buffer.write(line)
+        sys.stderr.flush()
+        if "id" not in experiment:
+            match = EXPERIMENT_LINE.match(line)
+            if match:
+                experiment["id"] = match[1].decode()
+
+
+def write_result(summary):
+    # Base64, because GitHub masks every line of a multi-line secret in logs and
+    # annotations, so a JSON credential alone turns each { and } into ***.
+    message = base64.b64encode(json.dumps(summary).encode()).decode()
+    print(f"::notice title={RESULT_TITLE}::{message}", flush=True)
+    lines = [
+        f"Test run: `{summary['testRunId']}`",
+        f"Commit: `{summary['commitSha']}`",
+    ]
+    if "stoppedEarly" in summary:
+        lines.append(
+            f"Stopped early: {summary['stoppedEarly']}. Traces that finished are saved in this test run."
+        )
+    else:
+        lines.append(f"Replayed: {summary['replayed']}, errored: {summary['errored']}")
+    with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as file:
+        file.write("### Bitfab replay\n\n" + "\n\n".join(lines) + "\n")
+
+
+def run_replay(root, config, request, args, timeout, experiment):
     with tempfile.TemporaryFile() as output:
         with subprocess.Popen(
             args,
             cwd=within(root, config["workingDirectory"]),
             stdout=output,
+            stderr=subprocess.PIPE,
             stdin=subprocess.DEVNULL,
             start_new_session=True,
         ) as child:
+            reader = threading.Thread(
+                target=forward_stderr, args=(child.stderr, experiment), daemon=True
+            )
+            reader.start()
             try:
-                deadline = time.monotonic() + 25 * 60
+                # Without --cloud-timeout the job's own timeout is the only limit.
+                deadline = None if timeout is None else time.monotonic() + timeout * 60
                 while child.poll() is None:
                     if os.fstat(output.fileno()).st_size > 16 * 1024 * 1024:
                         raise ValueError("Replay output exceeded 16 MiB")
-                    if time.monotonic() >= deadline:
-                        raise subprocess.TimeoutExpired(args, 25 * 60)
+                    if deadline is not None and time.monotonic() >= deadline:
+                        raise ValueError(
+                            f"Replay stopped at the {timeout}-minute --cloud-timeout; traces that finished are saved in Bitfab as an interrupted experiment"
+                        )
                     time.sleep(0.1)
                 code = child.returncode
             except BaseException:
                 with contextlib.suppress(ProcessLookupError):
                     os.killpg(child.pid, signal.SIGTERM)
+                # Time for the SDK to mark the experiment interrupted before the hard kill.
                 with contextlib.suppress(subprocess.TimeoutExpired):
-                    child.wait(timeout=10)
+                    child.wait(timeout=30)
                 with contextlib.suppress(ProcessLookupError):
                     os.killpg(child.pid, signal.SIGKILL)
                 child.wait()
                 raise
+            finally:
+                reader.join(timeout=5)
         if output.tell() > 16 * 1024 * 1024:
             raise ValueError("Replay output exceeded 16 MiB")
         output.seek(0)
@@ -843,14 +1002,6 @@ def execute():
         "replayed": len(items),
         "errored": sum(1 for item in items if item_errored(item)),
     }
-    (Path(os.environ["RUNNER_TEMP"]) / "bitfab-cloud-result.json").write_text(
-        json.dumps(summary) + "\n"
-    )
-    with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as file:
-        file.write(
-            f"### Bitfab replay\n\nTest run: `{test_run}`\n\nCommit: `{os.environ['GITHUB_SHA']}`\n\n"
-            f"Replayed: {summary['replayed']}, errored: {summary['errored']}\n"
-        )
     return summary
 
 
@@ -892,18 +1043,77 @@ def report_errored_items(record):
     return 0
 
 
+def replay_step(config, cli_command, env):
+    return {
+        "name": "Replay",
+        "working-directory": config["workingDirectory"],
+        "run": shlex.join([*cli_command, "--cloud-execute"]),
+        "env": {**env, **RUNNER_ENV},
+    }
+
+
+def workflow_document(job_options, setup_steps, replay):
+    """Only settings belong here; everything Bitfab may need to change runs inside the SDK."""
+    job = {
+        "runs-on": job_options.get("runsOn", "ubuntu-24.04"),
+        "steps": [
+            {
+                "name": "Check out replay snapshot",
+                "uses": "actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
+                "with": {"ref": "${{ github.sha }}", "persist-credentials": False},
+            },
+            *setup_steps,
+            replay,
+        ],
+    }
+    for key in ("environment", "services"):
+        if key in job_options:
+            job[key] = job_options[key]
+    return {
+        "name": "Bitfab cloud replay",
+        "run-name": RUN_NAME,
+        "on": {
+            "workflow_dispatch": {
+                "inputs": {
+                    "execution_id": {"required": True, "type": "string"},
+                    "request": {"required": True, "type": "string"},
+                }
+            }
+        },
+        "permissions": {"contents": "read"},
+        "jobs": {"replay": job},
+    }
+
+
+def write_new_files(root, outputs):
+    for name in outputs:
+        if within(root, name).exists():
+            raise ValueError(
+                f"Refusing to overwrite {name}; review and edit the existing setup"
+            )
+    for name, content in outputs.items():
+        target = within(root, name)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with target.open("x") as file:
+            file.write(content)
+
+
 def initialize(argv):
     parser = argparse.ArgumentParser(
         prog="bitfab-replay --cloud-init",
-        description="Install direct GitHub cloud replay from a reviewed JSON setup specification. Existing files are never overwritten.",
+        description="Install direct GitHub cloud replay from a reviewed JSON setup specification, or bring an existing setup up to date in place. Safe to run again.",
     )
     parser.add_argument(
         "--config",
-        required=True,
-        help="JSON file with cloud config, cliCommand, setupSteps, optional pipeline (omit or null to allow every registry pipeline), secrets, variables, environment, services, runsOn",
+        help="JSON file with cloud config, cliCommand, setupSteps, optional pipeline (omit or null to allow every registry pipeline), secrets, variables, environment, services, runsOn. Required for a new setup; for an existing one only cliCommand is read, and only when the workflow does not already show it",
     )
     args = parser.parse_args(argv)
-    spec = json.loads(Path(args.config).read_text())
+    spec = json.loads(Path(args.config).read_text()) if args.config else None
+    root = root_directory()
+    if within(root, CONFIG).exists():
+        return update_existing(root, spec)
+    if spec is None:
+        raise ValueError(f"No {CONFIG} yet; pass --config with a setup specification")
     config = {
         key: spec[key]
         for key in (
@@ -925,6 +1135,7 @@ def initialize(argv):
     config["secrets"] = secrets
     config["secretPrefix"] = secret_prefix
     validate_config(config)
+    validate_cli_command(spec.get("cliCommand"))
     if set(secrets) & set(variables):
         raise ValueError("Secret names and variable names must not overlap")
     if "BITFAB_API_KEY" not in secrets or not all(
@@ -933,16 +1144,6 @@ def initialize(argv):
         raise ValueError(
             "Supply uppercase secret/variable names, including BITFAB_API_KEY; never values"
         )
-    cli_command = spec.get("cliCommand")
-    if (
-        not isinstance(cli_command, list)
-        or not cli_command
-        or not all(isinstance(v, str) and v and "\x00" not in v for v in cli_command)
-        or any(v.startswith("--cloud") for v in cli_command)
-    ):
-        raise ValueError(
-            "cliCommand must be the argument array that starts the SDK's bitfab-replay command from workingDirectory"
-        )
     steps = spec.get("setupSteps")
     if (
         not isinstance(steps, list)
@@ -952,74 +1153,15 @@ def initialize(argv):
         raise ValueError("setupSteps must contain reviewed GitHub Actions setup steps")
     env = {key: "${{ secrets." + secret_prefix + key + " }}" for key in secrets}
     env.update({key: "${{ vars." + key + " }}" for key in variables})
-    env.update(
-        BITFAB_CLOUD_REQUEST="${{ inputs.request }}",
-        BITFAB_EXECUTION_ID="${{ inputs.execution_id }}",
-        BITFAB_COMMIT_SHA="${{ github.sha }}",
+    workflow = workflow_document(
+        spec, steps, replay_step(config, spec["cliCommand"], env)
     )
-    job = {
-        "runs-on": spec.get("runsOn", "ubuntu-24.04"),
-        "timeout-minutes": 35,
-        "steps": [
-            {
-                "name": "Check out replay snapshot",
-                "uses": "actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
-                "with": {"ref": "${{ github.sha }}", "persist-credentials": False},
-            },
-            *steps,
-            {
-                "name": "Replay",
-                "working-directory": config["workingDirectory"],
-                "run": shlex.join([*cli_command, "--cloud-execute"]),
-                "env": env,
-            },
-            {
-                "name": "Save replay identity",
-                "if": "always()",
-                "uses": "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02",
-                "with": {
-                    "name": "bitfab-replay-${{ inputs.execution_id }}",
-                    "path": "${{ runner.temp }}/bitfab-cloud-result.json",
-                    "if-no-files-found": "ignore",
-                    "retention-days": 7,
-                },
-            },
-        ],
-    }
-    for key in ("environment", "services"):
-        if key in spec:
-            job[key] = spec[key]
-    workflow = {
-        "name": "Bitfab cloud replay",
-        "run-name": "Bitfab replay ${{ inputs.execution_id }}",
-        "on": {
-            "workflow_dispatch": {
-                "inputs": {
-                    "execution_id": {"required": True, "type": "string"},
-                    "request": {"required": True, "type": "string"},
-                }
-            }
-        },
-        "permissions": {"contents": "read"},
-        "jobs": {"replay": job},
-    }
-    root = root_directory()
     outputs = {
         CONFIG: json.dumps(config, indent=2) + "\n",
         f".github/workflows/{config['workflow']}": json.dumps(workflow, indent=2)
         + "\n",
     }
-    for name in outputs:
-        target = within(root, name)
-        if target.exists():
-            raise ValueError(
-                f"Refusing to overwrite {name}; review and edit the existing setup"
-            )
-    for name, content in outputs.items():
-        target = within(root, name)
-        target.parent.mkdir(parents=True, exist_ok=True)
-        with target.open("x") as file:
-            file.write(content)
+    write_new_files(root, outputs)
     return {
         "files": list(outputs),
         "requiredSecrets": [secret_prefix + name for name in secrets],
@@ -1028,6 +1170,108 @@ def initialize(argv):
     }
 
 
+def replace_file(target, content):
+    with tempfile.NamedTemporaryFile(mode="w", dir=target.parent, delete=False) as file:
+        file.write(content)
+        temporary = file.name
+    os.replace(temporary, target)
+
+
+def update_existing(root, spec):
+    """Existing files win: only the Replay step and steps older setups generated change."""
+    config = json.loads(within(root, CONFIG).read_text())
+    validate_config(config)
+    name = f".github/workflows/{config['workflow']}"
+    workflow_path = within(root, name)
+    text = workflow_path.read_text() if workflow_path.is_file() else ""
+    try:
+        workflow = json.loads(text)
+        job = workflow["jobs"]["replay"]
+        steps = job["steps"]
+    except (ValueError, KeyError, TypeError) as error:
+        # A workflow the customer rewrote as YAML is current when nothing older remains.
+        if (
+            "--cloud-execute" in text
+            and OLD_UPLOAD_STEP not in text
+            and ".bitfab/cloudReplay.py" not in text
+            and not re.search(rf"timeout-minutes:\s*{OLD_JOB_TIMEOUT}\b", text)
+        ):
+            return {"files": [], "updated": False, "next": "Already up to date"}
+        raise ValueError(
+            f"{name} is missing or was not generated by --cloud-init; edit its Replay step by hand to run the SDK's bitfab-replay command with --cloud-execute"
+        ) from error
+    replay = [
+        index
+        for index, step in enumerate(steps)
+        if isinstance(step, dict) and step.get("name") == "Replay"
+    ]
+    if len(replay) != 1:
+        raise ValueError(f"{name} must contain exactly one step named Replay")
+    step = steps[replay[0]]
+    words = shlex.split(step.get("run", ""))
+    if spec is not None and spec.get("cliCommand") is not None:
+        cli_command = spec["cliCommand"]
+    elif words[-1:] == ["--cloud-execute"]:
+        cli_command = words[:-1]
+    elif config.get("cliCommand") is not None:
+        cli_command = config["cliCommand"]
+    else:
+        raise ValueError(
+            "The Replay step does not run the SDK command; pass --config with a file whose cliCommand starts bitfab-replay from workingDirectory"
+        )
+    validate_cli_command(cli_command)
+    env = {
+        key: value
+        for key, value in (step.get("env") or {}).items()
+        if key not in RUNNER_ENV
+    }
+    extra = {
+        key: value
+        for key, value in step.items()
+        if key not in ("name", "uses", "with", "run", "working-directory", "env")
+    }
+    before = json.dumps([config, workflow], sort_keys=True)
+    # Edit in place so the customer's checkout options, extra steps, and job
+    # settings survive; only the Replay step and our old result upload change.
+    job["steps"] = [
+        {**replay_step(config, cli_command, env), **extra}
+        if index == replay[0]
+        else entry
+        for index, entry in enumerate(steps)
+        if not (
+            isinstance(entry, dict)
+            and entry.get("name") == OLD_UPLOAD_STEP
+            and str(entry.get("uses", "")).startswith("actions/upload-artifact@")
+        )
+    ]
+    # The limit older setups generated; a value the customer chose stays.
+    if job.get("timeout-minutes") == OLD_JOB_TIMEOUT:
+        del job["timeout-minutes"]
+    config.pop("cliCommand", None)
+    # Setups before the SDK carried the script left a copy that nothing reads now.
+    old_script = ".bitfab/cloudReplay.py"
+    removable = [old_script] if within(root, old_script).exists() else []
+    if json.dumps([config, workflow], sort_keys=True) == before:
+        return {
+            "files": [],
+            "updated": False,
+            "removable": removable,
+            "next": "Already up to date",
+        }
+    outputs = {
+        CONFIG: json.dumps(config, indent=2) + "\n",
+        name: json.dumps(workflow, indent=2) + "\n",
+    }
+    for output, content in outputs.items():
+        replace_file(within(root, output), content)
+    return {
+        "files": list(outputs),
+        "updated": True,
+        "removable": removable,
+        "next": "Review the diff, merge the workflow to the default branch, then run --cloud-dry-run",
+    }
+
+
 def main():
     try:
         if sys.argv[1:2] == ["--cloud-init"]:
@@ -1040,6 +1284,11 @@ def main():
             result = run_cli(sys.argv[1:])
         print(json.dumps(result, indent=2))
         if result.get("state") == "completed" and result.get("conclusion") != "success":
+            if result.get("testRunId"):
+                print(
+                    f"Cloud replay stopped early ({result.get('stoppedEarly', result.get('conclusion'))}). Traces that finished are saved in test run {result['testRunId']}.",
+                    file=sys.stderr,
+                )
             return 1
         if result.get("state") == "completed":
             return report_errored_items(result)

data/lib/bitfab/cloud_replay_cli.rb

@@ -6,7 +6,7 @@ require "tempfile"
 
 module Bitfab
   module CloudReplayCli
-    HELP = "Direct GitHub replay: --cloud PIPELINE --trace-ids UUID[,UUID] [--registry PATH] [--max-concurrency 1..32] [--cloud-include FILE] [--cloud-dry-run] [--cloud-detach] [--cloud-request-id UUID]. Lifecycle: --cloud-status|--cloud-watch|--cloud-cancel|--cloud-cleanup UUID. Setup: --cloud-init --config FILE, --cloud-secrets --env-file FILE [NAME ...]. Requires git, gh auth login, Python 3.10+, and bitfab:setup cloud."
+    HELP = "Direct GitHub replay: --cloud PIPELINE --trace-ids UUID[,UUID] [--registry PATH] [--max-concurrency 1..32] [--cloud-include FILE] [--cloud-dry-run] [--cloud-detach] [--cloud-request-id UUID] [--cloud-timeout MINUTES]. Lifecycle: --cloud-status|--cloud-watch|--cloud-cancel|--cloud-cleanup UUID. Setup: --cloud-init [--config FILE] (creates a setup or updates it in place), --cloud-secrets --env-file FILE [NAME ...]. Requires git, gh auth login, Python 3.10+, and bitfab:setup cloud."
 
     HELPER = File.expand_path("cloudReplay.py", __dir__)
 

data/lib/bitfab/experiments.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "uri"
+
+module Bitfab
+  class Experiments
+    PATH = "/api/sdk/experiments"
+    private_constant :PATH
+
+    def initialize(http_client)
+      @http_client = http_client
+    end
+
+    def list(dataset_id: nil, trace_function_key: nil, git_branch: nil, experiment_group_id: nil, status: nil,
+      metadata: nil, created_after: nil, created_before: nil, cursor: nil, limit: nil)
+      params = {}
+      params["datasetId"] = dataset_id unless dataset_id.nil?
+      params["traceFunctionKey"] = trace_function_key unless trace_function_key.nil?
+      params["gitBranch"] = git_branch unless git_branch.nil?
+      params["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
+      params["status"] = status unless status.nil?
+      params["metadata"] = JSON.generate(metadata) unless metadata.nil?
+      params["createdAfter"] = iso8601(created_after) unless created_after.nil?
+      params["createdBefore"] = iso8601(created_before) unless created_before.nil?
+      params["cursor"] = cursor unless cursor.nil?
+      params["limit"] = limit unless limit.nil?
+      query = URI.encode_www_form(params)
+      @http_client.get(query.empty? ? PATH : "#{PATH}?#{query}")
+    end
+
+    def get(id)
+      @http_client.get("#{PATH}/#{encode(id)}")["experiment"]
+    end
+
+    def get_rollup(id)
+      @http_client.get("#{PATH}/#{encode(id)}/rollup")
+    end
+
+    def get_rollup_all(ids)
+      return [] if ids.empty?
+
+      query = URI.encode_www_form("experimentIds" => ids.join(","))
+      @http_client.get("#{PATH}/rollups?#{query}")["rollups"]
+    end
+
+    private
+
+    def iso8601(value)
+      value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+    end
+
+    def encode(value)
+      URI.encode_www_form_component(value).gsub("+", "%20")
+    end
+  end
+end

data/lib/bitfab/http_client.rb

@@ -261,7 +261,7 @@ module Bitfab
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, include_db_branch_lease: false, dataset_ids: nil,
+      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -272,6 +272,7 @@ module Bitfab
       payload["traceIds"] = trace_ids if trace_ids
       payload["name"] = name unless name.nil?
       payload["notes"] = notes unless notes.nil?
+      payload["metadata"] = metadata unless metadata.nil?
       payload["codeChangeDescription"] = code_change_description unless code_change_description.nil?
       payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?

data/lib/bitfab/replay.rb

@@ -280,7 +280,7 @@ module Bitfab
     #   crashes the run.
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
-    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil,
+    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
@@ -378,6 +378,7 @@ module Bitfab
         trace_ids:,
         name:,
         notes:,
+        metadata:,
         code_change_description:,
         code_change_files:,
         experiment_group_id:,

data/lib/bitfab/replay_registry.rb

@@ -19,7 +19,7 @@ module Bitfab
   # Project-owned registry of production replay roots.
   class ReplayRegistry
     OPTION_NAMES = %i[
-      limit trace_ids name notes concurrency max_concurrency code_change_description
+      limit trace_ids name notes metadata concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
     ].freeze
@@ -186,7 +186,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name notes max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions judge_assertions dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions judge_assertions dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -291,6 +291,15 @@ module Bitfab
         options.on("--notes TEXT",
           "Run conditions Bitfab cannot see on its own, such as an environment override or a forced feature flag. " \
             "Kept on the experiment next to its name.") { |value| args[:notes] = value }
+        options.on("--metadata KEY=VALUE",
+          "A key and value stored on the experiment, such as schedule=eod. Repeat the flag for more pairs. " \
+            "Filter later runs on it with client.experiments.list(metadata: {...}).") do |value|
+          key, pair_value = value.split("=", 2)
+          if key.to_s.empty? || pair_value.nil?
+            raise OptionParser::InvalidArgument, "--metadata expects KEY=VALUE, got '#{value}'"
+          end
+          (args[:metadata] ||= {})[key] = pair_value
+        end
         options.on("--concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--concurrency", value)
         end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.62.3"
+  VERSION = "0.62.5"
 end

data/lib/bitfab.rb

@@ -21,6 +21,7 @@ require_relative "bitfab/traces"
 require_relative "bitfab/assertions"
 require_relative "bitfab/labels"
 require_relative "bitfab/graders"
+require_relative "bitfab/experiments"
 require_relative "bitfab/mock_override"
 require_relative "bitfab/replay"
 require_relative "bitfab/replay_branch"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.62.3
+  version: 0.62.5
 platform: ruby
 authors:
 - Harvest Team
@@ -170,6 +170,7 @@ files:
 - lib/bitfab/detached_trace.rb
 - lib/bitfab/env.rb
 - lib/bitfab/experiment_id.rb
+- lib/bitfab/experiments.rb
 - lib/bitfab/git_command.rb
 - lib/bitfab/git_state.rb
 - lib/bitfab/graders.rb