bitfab 0.64.2 → 0.64.4

package/cloudReplay.py

@@ -40,6 +40,7 @@ OUTPUT_BEGIN = "bitfab-replay-output-begin"
 OUTPUT_END = "bitfab-replay-output-end"
 LOG_TAIL_LINES = 40
 OUTPUT_LIMIT = 16 * 1024 * 1024
+DISK_LIMIT = 1024 * 1024 * 1024
 NEW_FILES_LIMIT = 20 * 1024 * 1024
 POLL_SECONDS = 5
 ITEM_ERROR_FIELDS = (
@@ -52,6 +53,14 @@ ITEM_ERROR_FIELDS = (
 ITEM_ID_FIELDS = ("originalTraceId", "original_trace_id", "traceId", "trace_id")
 ITEM_ERROR_LINES = 20
 ITEM_ERROR_LENGTH = 500
+SUMMARY_ERROR_ITEMS = 100
+EXPERIMENT_FIELDS = {
+    "experimentId": ("experimentId", "experiment_id"),
+    "experimentUrl": ("experimentUrl", "experiment_url"),
+    "testRunId": ("testRunId", "test_run_id"),
+    "testRunUrl": ("testRunUrl", "test_run_url"),
+    "attempts": ("attempts",),
+}
 UUID = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$")
 SHA = re.compile(r"^[0-9a-f]{40}$")
 GITHUB_REMOTE = re.compile(
@@ -59,6 +68,15 @@ GITHUB_REMOTE = re.compile(
 )
 LOG_TIMESTAMP = re.compile(r"^?\d{4}-\d\d-\d\dT[\d:.]+Z ?")
 EXPERIMENT_LINE = re.compile(rb"^\[replay\] Experiment ([0-9a-f-]{36}):")
+PROGRESS_PREFIX = b"@@bitfab:progress "
+PROGRESS_PAYLOAD_FIELDS = (
+    "input",
+    "result",
+    "originalOutput",
+    "original_output",
+    "selectiveReplay",
+    "selective_replay",
+)
 SECRET_REFERENCE = re.compile(
     r"[\"']?([A-Za-z_][A-Za-z0-9_]*)[\"']?[ \t]*:[ \t]*[\"']?\$\{\{\s*secrets(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[\s*[\"']([A-Za-z_][A-Za-z0-9_]*)[\"']\s*\])\s*\}\}"
 )
@@ -77,7 +95,7 @@ FOLLOW_FLAGS = (
     "--cloud-cleanup",
 )
 CLOUD_VALUE_FLAGS = (*FOLLOW_FLAGS, "--cloud-request-id", "--cloud-timeout")
-CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach")
+CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach", "--cloud-full-output")
 RENAMED_FLAGS = {
     "--cloud-dry-run": "--cloud-preview",
     "--cloud-check": "--dry-run",
@@ -113,9 +131,17 @@ does not ignore) from the same directory, and this command prints the replay's o
 and exits with its exit code. Credential-like files are refused. Requires git and
 Python 3.10+ on macOS or Linux, with a github.com origin.
 
+The replay's stderr comes back as it printed it. Its stdout result comes back as a
+short summary instead of every item: the experiment ID and URL, how many traces were
+replayed, same, changed, and errored, each errored trace with its error, and the exit
+code. Every item's input and output are already saved in the experiment in Bitfab.
+
 Options added by --cloud:
   --cloud-preview        List what the snapshot would contain and push nothing.
   --cloud-detach         Return after dispatch instead of waiting.
+  --cloud-full-output    Print the replay's whole stdout result, every item included,
+                         instead of the summary. Large runs can exceed what the
+                         GitHub job log holds.
   --cloud-timeout MIN    Stop the replay after MIN minutes (1..7200).
   --cloud-request-id ID  Recover a submission whose response was lost.
 
@@ -310,6 +336,16 @@ def parse_environment_file(text):
     return values
 
 
+def secret_value(value):
+    if "\n" not in value:
+        return value
+    try:
+        parsed = json.loads(value)
+    except ValueError:
+        return value
+    return json.dumps(parsed, separators=(",", ":"))
+
+
 def ensure_environment(repo, environment):
     try:
         api(repo, f"environments/{environment}")
@@ -412,7 +448,7 @@ def configure_secrets(argv):
                 repo,
                 *(["--env", environment] if environment else []),
             ],
-            input=values.get(name, values.get(targets[name])),
+            input=secret_value(values.get(name, values.get(targets[name]))),
             env={**os.environ, "GH_TOKEN": token},
         )
     result["set"] = [targets[name] for name in found]
@@ -592,10 +628,8 @@ def validate_arguments(args):
     if not isinstance(args, list) or not all(isinstance(value, str) for value in args):
         raise ValueError("Replay arguments must be a list of strings")
     for value in args:
-        if "\x00" in value or "\n" in value or len(value) > 1000:
-            raise ValueError(
-                "Replay options must be single-line values of at most 1000 characters"
-            )
+        if "\x00" in value or "\n" in value:
+            raise ValueError("Replay options must be single-line values")
         if value.startswith("--cloud"):
             raise ValueError("The runner replays locally and never dispatches again")
     if sum(len(value) for value in args) > 16000:
@@ -639,6 +673,8 @@ def replay_request(root, parsed):
     }
     if "--cloud-timeout" in parsed["cloud"]:
         request["timeoutMinutes"] = int(parsed["cloud"]["--cloud-timeout"])
+    if "--cloud-full-output" in parsed["switches"]:
+        request["fullOutput"] = True
     return request, files
 
 
@@ -862,9 +898,7 @@ def outcome(record):
         raise ValueError("Replay result does not match this execution")
     record["exitCode"] = result["exitCode"]
     parsed = last_json(result.get("stdout", "")) or {}
-    test_run = result.get("testRunId") or parsed.get(
-        "testRunId", parsed.get("test_run_id")
-    )
+    test_run = result.get("testRunId") or experiment_fields(parsed).get("testRunId")
     if isinstance(test_run, str) and UUID.fullmatch(test_run):
         record["testRunId"] = test_run
     if result.get("stoppedEarly"):
@@ -889,7 +923,22 @@ def print_outcome(found):
     for line in replay_output(lines) or []:
         print(line, file=sys.stderr)
     sys.stderr.flush()
+    if result.get("stdoutOmittedBytes"):
+        progress(
+            f"The replay also printed {result['stdoutOmittedBytes'] // (1024 * 1024)} MiB to stdout before its result, such as a library's logging; only the end was kept"
+        )
     print(result.get("stdout", ""), end="", flush=True)
+    summary = last_json(result.get("stdout", "")) or {}
+    if summary.get("cloudSummary") is True:
+        progress(
+            "Printed a summary of the result; every item's input and output are in the experiment"
+            + (
+                f" at {summary['experimentUrl']}"
+                if summary.get("experimentUrl")
+                else ""
+            )
+            + ". Add --cloud-full-output to print every item here instead"
+        )
     if record.get("stoppedEarly"):
         progress(
             f"Replay stopped early: {record['stoppedEarly']}. Traces that finished are saved"
@@ -1049,7 +1098,10 @@ def submit(root, repo, workflow, parsed, path):
 def report_steps(record, shown):
     job = replay_job(record)
     for step in (job or {}).get("steps", []):
-        if step.get("status") != "queued" and step.get("name") not in shown:
+        if (
+            step.get("status") in ("in_progress", "completed")
+            and step.get("name") not in shown
+        ):
             shown.add(step.get("name"))
             progress(step.get("name"))
 
@@ -1186,8 +1238,12 @@ def execute():
     previous = signal.signal(signal.SIGTERM, raise_interrupt)
     print(OUTPUT_BEGIN, flush=True)
     try:
-        code, stdout, stopped = run_command(
-            directory, args, request.get("timeoutMinutes"), experiment
+        code, stdout, omitted, stopped = run_command(
+            directory,
+            args,
+            request.get("timeoutMinutes"),
+            experiment,
+            summarize=request.get("fullOutput") is not True,
         )
     finally:
         print(OUTPUT_END, flush=True)
@@ -1198,6 +1254,8 @@ def execute():
         "exitCode": code,
         "stdout": stdout,
     }
+    if omitted:
+        result["stdoutOmittedBytes"] = omitted
     if stopped:
         result["stoppedEarly"] = stopped
         if UUID.fullmatch(experiment.get("id", "")):
@@ -1224,6 +1282,15 @@ def check_secrets(root):
             + ", ".join(f"{name} (secret {targets[name]})" for name in empty)
             + ". GitHub passes a secret that does not exist as an empty string. Create each one under Settings, Secrets and variables, Actions, in the repository or in the job's Environment"
         )
+    for name in targets:
+        if any(
+            0 < len(line.strip()) < 4
+            for line in os.environ.get(name, "").splitlines()[1:]
+        ):
+            print(
+                f"::warning title=Bitfab replay::{name} has a line of three characters or fewer, such as a lone brace in multi-line JSON, and GitHub masks that text everywhere in this log. Store it on one line; bitfab-replay --cloud-secrets does this for JSON",
+                flush=True,
+            )
 
 
 def run_check_command(directory):
@@ -1259,10 +1326,75 @@ def item_errors(items):
         text = " ".join(str(error).split())
         if len(text) > ITEM_ERROR_LENGTH:
             text = text[:ITEM_ERROR_LENGTH] + "..."
-        errors.append((trace, text))
+        errored = {"originalTraceId": trace, "error": text}
+        if isinstance(item.get("attempt"), int):
+            errored["attempt"] = item["attempt"]
+        errors.append(errored)
     return errors
 
 
+def item_field(item, camel, snake):
+    return item[camel] if camel in item else item.get(snake)
+
+
+def canonical(value):
+    return json.dumps(value, separators=(",", ":"))
+
+
+def experiment_fields(parsed):
+    fields = {}
+    for name, spellings in EXPERIMENT_FIELDS.items():
+        value = next(
+            (parsed[key] for key in spellings if parsed.get(key) is not None), None
+        )
+        if value is not None:
+            fields[name] = value
+    return fields
+
+
+def replay_summary(parsed):
+    items = [item for item in parsed["items"] if isinstance(item, dict)]
+    replayed = [item for item in items if not carried_over(item)]
+    errors = item_errors(replayed)
+    counts = {
+        "replayed": len(replayed),
+        "carriedOver": len(items) - len(replayed),
+        "same": 0,
+        "changed": 0,
+        "matchedExpected": 0,
+        "missedExpected": 0,
+        "errored": len(errors),
+    }
+    for item in replayed:
+        if item_errored(item):
+            continue
+        equal = canonical(item.get("result")) == canonical(
+            item_field(item, "originalOutput", "original_output")
+        )
+        if item_field(item, "ingestionType", "ingestion_type") == "seeded":
+            counts["matchedExpected" if equal else "missedExpected"] += 1
+        else:
+            counts["same" if equal else "changed"] += 1
+    summary = {
+        "cloudSummary": True,
+        **experiment_fields(parsed),
+        "counts": counts,
+        "erroredItems": errors[:SUMMARY_ERROR_ITEMS],
+    }
+    if len(errors) > SUMMARY_ERROR_ITEMS:
+        summary["erroredItemsOmitted"] = len(errors) - SUMMARY_ERROR_ITEMS
+    return summary
+
+
+def result_summary(stdout):
+    parsed = last_json(stdout) or {}
+    if parsed.get("cloudSummary") is True:
+        return parsed
+    if isinstance(parsed.get("items"), list):
+        return replay_summary(parsed)
+    return None
+
+
 def item_errored(item):
     return isinstance(item, dict) and any(
         item.get(field) is not None for field in ITEM_ERROR_FIELDS
@@ -1279,8 +1411,24 @@ def raise_interrupt(signum, frame):
     raise KeyboardInterrupt
 
 
-def forward_stderr(stream, experiment):
+def without_item_payload(line):
+    try:
+        event = json.loads(line[len(PROGRESS_PREFIX) :])
+    except ValueError:
+        return line
+    item = event.get("item") if isinstance(event, dict) else None
+    if not isinstance(item, dict):
+        return line
+    event["item"] = {
+        key: value for key, value in item.items() if key not in PROGRESS_PAYLOAD_FIELDS
+    }
+    return PROGRESS_PREFIX + json.dumps(event).encode() + b"\n"
+
+
+def forward_stderr(stream, experiment, summarize):
     for line in iter(stream.readline, b""):
+        if summarize and line.startswith(PROGRESS_PREFIX):
+            line = without_item_payload(line)
         sys.stdout.write(line.decode(errors="replace"))
         sys.stdout.flush()
         if "id" not in experiment:
@@ -1289,18 +1437,40 @@ def forward_stderr(stream, experiment):
                 experiment["id"] = match[1].decode()
 
 
-def last_json(text):
+def trailing_json(text):
     decoder = json.JSONDecoder()
-    result = None
     for offset in [0, *[i + 1 for i, value in enumerate(text) if value == "\n"]]:
         if text.startswith("{", offset):
             try:
                 value, end = decoder.raw_decode(text, offset)
-                if not text[end:].strip() and isinstance(value, dict):
-                    result = value
             except json.JSONDecodeError:
-                pass
-    return result
+                continue
+            if not text[end:].strip() and isinstance(value, dict):
+                return offset, value
+    return None, None
+
+
+def last_json(text):
+    return trailing_json(text)[1]
+
+
+def relayed_stdout(output, code, summarize):
+    if summarize:
+        output.seek(0)
+        parsed = last_json(output.read().decode(errors="replace"))
+        if isinstance(parsed, dict) and isinstance(parsed.get("items"), list):
+            return json.dumps({**replay_summary(parsed), "exitCode": code}) + "\n", 0
+    return carried_stdout(output)
+
+
+def carried_stdout(output):
+    start = max(0, os.fstat(output.fileno()).st_size - OUTPUT_LIMIT)
+    output.seek(start)
+    text = output.read().decode(errors="replace")
+    if not start:
+        return text, 0
+    offset = trailing_json(text)[0] or 0
+    return text[offset:], start + len(text[:offset].encode())
 
 
 def write_result(result):
@@ -1314,35 +1484,45 @@ def write_result(result):
             file.write(summary_markdown(result))
 
 
+def annotate(result):
+    if result.get("stoppedEarly"):
+        message = f"Replay stopped early: {result['stoppedEarly']}"
+    elif result["exitCode"]:
+        message = f"Replay exited {result['exitCode']}; bitfab-replay --cloud reports it as its own exit code"
+    else:
+        return
+    print(f"::warning title=Bitfab replay::{message}", flush=True)
+
+
 def summary_markdown(result):
-    parsed = last_json(result["stdout"]) or {}
-    test_run = result.get("testRunId") or parsed.get(
-        "testRunId", parsed.get("test_run_id")
-    )
-    items = parsed.get("items") if isinstance(parsed.get("items"), list) else []
-    replayed = [item for item in items if not carried_over(item)]
-    errors = item_errors(replayed)
+    summary = result_summary(result["stdout"]) or {}
+    counts = summary.get("counts", {})
+    test_run = result.get("testRunId") or summary.get("testRunId")
+    errors = summary.get("erroredItems", [])
     lines = [f"Commit: `{result['commitSha']}`", f"Exit code: {result['exitCode']}"]
     if test_run:
         lines.insert(0, f"Test run: `{test_run}`")
     if result.get("stoppedEarly"):
         lines.append(f"Stopped early: {result['stoppedEarly']}")
-    elif items:
-        lines.append(f"Replayed: {len(replayed)}, errored: {len(errors)}")
+    elif counts.get("replayed") or counts.get("carriedOver"):
+        lines.append(
+            f"Replayed: {counts['replayed']}, same: {counts['same']}, changed: {counts['changed']}, errored: {counts['errored']}"
+        )
     text = "### Bitfab replay\n\n" + "\n\n".join(lines) + "\n"
     if errors:
         shown = [
-            f"trace {trace}: {error}" for trace, error in errors[:ITEM_ERROR_LINES]
+            f"trace {error['originalTraceId']}: {error['error']}"
+            for error in errors[:ITEM_ERROR_LINES]
         ]
-        if len(errors) > len(shown):
-            shown.append(f"and {len(errors) - len(shown)} more errored items")
+        if counts.get("errored", 0) > len(shown):
+            shown.append(f"and {counts['errored'] - len(shown)} more errored items")
         text += "\n#### Errored items\n\n" + "".join(
             f"- `{line.replace('`', chr(39))}`\n" for line in shown
         )
     return text
 
 
-def run_command(directory, args, timeout, experiment):
+def run_command(directory, args, timeout, experiment, *, summarize=True):
     stopped = None
     with tempfile.TemporaryFile() as output:
         with subprocess.Popen(
@@ -1354,18 +1534,26 @@ def run_command(directory, args, timeout, experiment):
             start_new_session=True,
         ) as child:
             reader = threading.Thread(
-                target=forward_stderr, args=(child.stderr, experiment), daemon=True
+                target=forward_stderr,
+                args=(child.stderr, experiment, summarize),
+                daemon=True,
             )
             reader.start()
             try:
                 deadline = None if timeout is None else time.monotonic() + timeout * 60
+                parent = os.getppid()
+                checked = time.monotonic()
                 while child.poll() is None:
-                    if os.fstat(output.fileno()).st_size > OUTPUT_LIMIT:
-                        raise ValueError("Replay output exceeded 16 MiB")
+                    if os.fstat(output.fileno()).st_size > DISK_LIMIT:
+                        raise ValueError("Replay printed more than 1 GiB to stdout")
                     if deadline is not None and time.monotonic() >= deadline:
                         stopped = f"it reached the {timeout}-minute --cloud-timeout"
                         stop(child)
                         break
+                    if time.monotonic() - checked >= 1:
+                        checked = time.monotonic()
+                        if os.getppid() != parent:
+                            raise KeyboardInterrupt
                     time.sleep(0.1)
             except KeyboardInterrupt:
                 stopped = "the GitHub run was cancelled"
@@ -1375,12 +1563,9 @@ def run_command(directory, args, timeout, experiment):
                 raise
             finally:
                 reader.join(timeout=5)
-            code = child.returncode
-        if output.tell() > OUTPUT_LIMIT:
-            raise ValueError("Replay output exceeded 16 MiB")
-        output.seek(0)
-        text = output.read().decode(errors="replace")
-    return (code if code >= 0 else 1), text, stopped
+            code = child.returncode if child.returncode >= 0 else 1
+        text, omitted = relayed_stdout(output, code, summarize)
+    return code, text, omitted, stopped
 
 
 def stop(child):
@@ -1772,8 +1957,8 @@ def main():
             print_json(configure_secrets(argv[1:]))
             return 0
         if argv == ["--cloud-execute"]:
-            result = execute()
-            return result["exitCode"] or (1 if result.get("stoppedEarly") else 0)
+            annotate(execute())
+            return 0
         if "-h" in argv or "--help" in argv:
             print(HELP, end="")
             return 0

package/dist/{chunk-GLWAET3G.js → chunk-I6B5R2FP.js}

@@ -6,7 +6,7 @@ import {
   replayContextReady,
   runWithReplayContext,
   unrefTimer
-} from "./chunk-XHUW3NY6.js";
+} from "./chunk-ZPVUSLIU.js";
 import {
   runWithCaptureSuppressed,
   warnOnce,
@@ -1884,4 +1884,4 @@ export {
   warnJudgeAssertionsDeprecated,
   replay
 };
-//# sourceMappingURL=chunk-GLWAET3G.js.map
+//# sourceMappingURL=chunk-I6B5R2FP.js.map

package/dist/{chunk-FX4X7LOW.js → chunk-OEHQPSOB.js}

@@ -2,7 +2,7 @@ import {
   BitfabError,
   warnOnce,
   writeOnce
-} from "./chunk-O632VPWV.js";
+} from "./chunk-XASGXGQN.js";
 
 // src/codeChange.ts
 var NUL = String.fromCharCode(0);
@@ -316,7 +316,7 @@ async function seedFromRegistry(registry, pipeline, cases, options = {}) {
       sessionId: seedCase.sessionId
     })
   );
-  const { flushTraces: flushTraces2 } = await import("./http-IOT2GDD2.js");
+  const { flushTraces: flushTraces2 } = await import("./http-Y6T6T2AP.js");
   await flushTraces2(3e4);
   return { pipeline, traceFunctionKey, traceIds };
 }
@@ -1007,4 +1007,4 @@ export {
   runRegistryMain,
   runIfEntrypoint
 };
-//# sourceMappingURL=chunk-FX4X7LOW.js.map
+//# sourceMappingURL=chunk-OEHQPSOB.js.map

package/dist/{chunk-ENORQO5K.js → chunk-UMZDRAAK.js}

@@ -17,7 +17,7 @@ import {
   serializeValue,
   toJsonSafe,
   toJsonSafeReport
-} from "./chunk-GLWAET3G.js";
+} from "./chunk-I6B5R2FP.js";
 import {
   BitfabError,
   CONTENT_OFF_KEY,
@@ -36,7 +36,7 @@ import {
   readEnv,
   recordCallerTraceMetadata,
   retireTraceMetadata
-} from "./chunk-XHUW3NY6.js";
+} from "./chunk-ZPVUSLIU.js";
 import {
   __privateAdd,
   __privateGet,
@@ -6253,7 +6253,7 @@ var Bitfab = class {
       await runWithSeedContext({ traceId }, async () => target(...args));
     } finally {
       try {
-        const { flushTraces: flushTraces2 } = await import("./http-MFXRJ6RS.js");
+        const { flushTraces: flushTraces2 } = await import("./http-53RWIBXJ.js");
         await flushTraces2(3e4);
       } finally {
         unrecorded = activeTraceStates.delete(traceId);
@@ -6312,7 +6312,7 @@ var Bitfab = class {
         `Function is wrapped with trace function key '${wrappedKey}' but replay was called with '${traceFunctionKey}'. Pass matching keys, or pass the unwrapped function to replay it under the explicit key.`
       );
     }
-    const { replay: doReplay } = await import("./replay-QZQYRTAO.js");
+    const { replay: doReplay } = await import("./replay-SIKHFPKA.js");
     return doReplay(
       this.httpClient,
       this.serviceUrl,
@@ -6658,7 +6658,7 @@ async function seedFromRegistry(registry, pipeline, cases, options = {}) {
       sessionId: seedCase.sessionId
     })
   );
-  const { flushTraces: flushTraces2 } = await import("./http-MFXRJ6RS.js");
+  const { flushTraces: flushTraces2 } = await import("./http-53RWIBXJ.js");
   await flushTraces2(3e4);
   return { pipeline, traceFunctionKey, traceIds };
 }
@@ -6703,4 +6703,4 @@ export {
   reseedFromRegistry,
   seedFromRegistry
 };
-//# sourceMappingURL=chunk-ENORQO5K.js.map
+//# sourceMappingURL=chunk-UMZDRAAK.js.map

package/dist/{chunk-O632VPWV.js → chunk-XASGXGQN.js}

@@ -181,7 +181,7 @@ function encodeRequestBody(body) {
 }
 
 // src/version.generated.ts
-var __version__ = "0.64.2";
+var __version__ = "0.64.4";
 var __packageName__ = "bitfab";
 
 // src/errors.ts
@@ -2500,4 +2500,4 @@ export {
   HttpClient,
   experimentIdWireFields
 };
-//# sourceMappingURL=chunk-O632VPWV.js.map
+//# sourceMappingURL=chunk-XASGXGQN.js.map