@bitfab/sdk 0.64.14 → 0.64.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cloudReplay.py +250 -55
- package/dist/{chunk-F5ZFL5W6.js → chunk-AXFKZYIY.js} +25 -6
- package/dist/chunk-AXFKZYIY.js.map +1 -0
- package/dist/{chunk-GURR5AWG.js → chunk-I4TFQZCW.js} +2 -2
- package/dist/{chunk-GURR5AWG.js.map → chunk-I4TFQZCW.js.map} +1 -1
- package/dist/{chunk-2EWCNKQL.js → chunk-LOIVELPI.js} +10 -5
- package/dist/{chunk-2EWCNKQL.js.map → chunk-LOIVELPI.js.map} +1 -1
- package/dist/{chunk-PXNFY22G.js → chunk-NNZAQSTW.js} +2 -2
- package/dist/{chunk-A7NX4OKG.js → chunk-PAWBCEI7.js} +2 -2
- package/dist/{chunk-A7NX4OKG.js.map → chunk-PAWBCEI7.js.map} +1 -1
- package/dist/{http-QX6O4EDS.js → http-LV7OUBKA.js} +2 -2
- package/dist/{http-C44ETCJK.js → http-QQ6VVHUZ.js} +2 -2
- package/dist/index.cjs +20 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +99 -2
- package/dist/index.d.ts +99 -2
- package/dist/index.js +3 -3
- package/dist/node.cjs +20 -1
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +3 -3
- package/dist/{replay-MFZF4SY7.js → replay-JNKXZPY7.js} +3 -3
- package/dist/replayCli.js +13 -2
- package/dist/replayCli.js.map +1 -1
- package/dist/seedCli.js +2 -2
- package/package.json +1 -1
- package/dist/chunk-F5ZFL5W6.js.map +0 -1
- /package/dist/{chunk-PXNFY22G.js.map → chunk-NNZAQSTW.js.map} +0 -0
- /package/dist/{http-C44ETCJK.js.map → http-LV7OUBKA.js.map} +0 -0
- /package/dist/{http-QX6O4EDS.js.map → http-QQ6VVHUZ.js.map} +0 -0
- /package/dist/{replay-MFZF4SY7.js.map → replay-JNKXZPY7.js.map} +0 -0
package/cloudReplay.py
CHANGED
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
4
|
import base64
|
|
5
|
+
import calendar
|
|
5
6
|
import contextlib
|
|
6
7
|
import functools
|
|
7
8
|
import json
|
|
@@ -47,6 +48,9 @@ OUTPUT_LIMIT = 16 * 1024 * 1024
|
|
|
47
48
|
DISK_LIMIT = 1024 * 1024 * 1024
|
|
48
49
|
NEW_FILES_LIMIT = 20 * 1024 * 1024
|
|
49
50
|
POLL_SECONDS = 5
|
|
51
|
+
CLEANABLE_STATES = ("completed", "prepared", "dispatch_unknown", "dispatch_rejected")
|
|
52
|
+
CLOCK_SKEW_SECONDS = 600
|
|
53
|
+
TIMESTAMP = "%Y-%m-%dT%H:%M:%SZ"
|
|
50
54
|
ITEM_ERROR_FIELDS = (
|
|
51
55
|
"error",
|
|
52
56
|
"traceError",
|
|
@@ -99,6 +103,8 @@ SECRET_ENV_NAME = re.compile(
|
|
|
99
103
|
r"(?:^|_)(?:KEY|SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIALS?)(?:_|$)", re.IGNORECASE
|
|
100
104
|
)
|
|
101
105
|
ENV_FLAG = "--cloud-env"
|
|
106
|
+
ALLOW_FLAG = "--cloud-allow-file"
|
|
107
|
+
REPEATED_FLAGS = (ENV_FLAG, ALLOW_FLAG)
|
|
102
108
|
URL_PASSWORD = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:[^/\s@]+@")
|
|
103
109
|
FOLLOW_FLAGS = (
|
|
104
110
|
"--cloud-status",
|
|
@@ -121,7 +127,10 @@ RENAMED_FLAGS = {
|
|
|
121
127
|
SEED_FLAGS = ("--seed", "--cases", "--from-trace", "--run")
|
|
122
128
|
PATH_FLAGS = ("--registry", "--params", "--code-change")
|
|
123
129
|
CODE_CHANGE_FLAGS = ("--code-change", "--no-code-change")
|
|
124
|
-
|
|
130
|
+
CHECK_ARGUMENTS_SECONDS = 300
|
|
131
|
+
CHECK_ARGUMENTS_FLAG = "--check-arguments"
|
|
132
|
+
REPLAY_ARGUMENTS_FLAG = "--replay-arguments"
|
|
133
|
+
ARGUMENTS_CHECKED_ENV = "BITFAB_REPLAY_ARGUMENTS_CHECKED"
|
|
125
134
|
ACTIONS = {
|
|
126
135
|
"checkout": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1",
|
|
127
136
|
"node": "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020",
|
|
@@ -142,35 +151,45 @@ HELP = """Cloud replay: run the replay you run locally on GitHub Actions by addi
|
|
|
142
151
|
|
|
143
152
|
bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud
|
|
144
153
|
|
|
145
|
-
Every replay option works as it does locally, including --dry-run and --fail-on-error
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
and
|
|
149
|
-
|
|
154
|
+
Every replay option works as it does locally, including --dry-run and --fail-on-error,
|
|
155
|
+
and your SDK checks them before anything is uploaded. The runner replays a snapshot of
|
|
156
|
+
your working tree (changed files, plus new files git does not ignore) from the same
|
|
157
|
+
directory, and this command prints the replay's output and exits with its exit code.
|
|
158
|
+
Credential-like files that are not on GitHub yet are refused unless you name them with
|
|
159
|
+
--cloud-allow-file. Requires git and Python 3.10+ on macOS or Linux, with a github.com
|
|
160
|
+
origin.
|
|
150
161
|
|
|
151
162
|
The replay's stderr comes back as it printed it. Its stdout result comes back as a
|
|
152
163
|
short summary instead of every item: the experiment ID and URL, how many traces were
|
|
153
164
|
replayed, same, changed, and errored, each errored trace with its error, and the exit
|
|
154
|
-
code. Every item's input and output are
|
|
165
|
+
code. Every item's input and output are saved in the experiment in Bitfab, where the
|
|
166
|
+
Bitfab MCP tools read them, so by default the job log holds no trace data.
|
|
155
167
|
|
|
156
168
|
Options added by --cloud:
|
|
157
169
|
--cloud-preview List what the snapshot would contain and push nothing.
|
|
158
170
|
--cloud-detach Return after dispatch instead of waiting.
|
|
159
171
|
--cloud-full-output Print the replay's whole stdout result, every item included,
|
|
160
|
-
instead of the summary.
|
|
161
|
-
GitHub job log
|
|
172
|
+
instead of the summary. The result passes through the
|
|
173
|
+
GitHub job log, so it is refused on a public repository,
|
|
174
|
+
and large runs can exceed what the log holds.
|
|
162
175
|
--cloud-timeout MIN Stop the replay after MIN minutes (1..7200).
|
|
163
176
|
--cloud-env NAME=VALUE Set an environment variable for this replay only; repeat it
|
|
164
177
|
for more. NAME alone copies the value from your shell. The
|
|
165
178
|
value is stored in the snapshot commit on GitHub, so this is
|
|
166
179
|
for settings, never secrets: names that look like secrets,
|
|
167
180
|
and names the workflow reads from a secret, are refused.
|
|
181
|
+
--cloud-allow-file PATH
|
|
182
|
+
Upload PATH even though it looks like a credential, such as
|
|
183
|
+
a test certificate; repeat it for more. Refused on a public
|
|
184
|
+
repository, where anyone can read the snapshot branch.
|
|
168
185
|
--cloud-request-id ID Recover a submission whose response was lost.
|
|
169
186
|
--cloud-ref BRANCH Replay BRANCH as it is on GitHub, with no snapshot of your
|
|
170
187
|
working tree, and record the experiment on BRANCH. Meant for
|
|
171
|
-
CI jobs, such as a nightly replay of dev.
|
|
172
|
-
|
|
173
|
-
|
|
188
|
+
CI jobs, such as a nightly replay of dev. Runs from a
|
|
189
|
+
checkout of the repository, with GitHub access that can
|
|
190
|
+
read it and run Actions (a job token needs contents: read
|
|
191
|
+
and actions: write, passed as GH_TOKEN), and a workflow on
|
|
192
|
+
BRANCH that declares a request input.
|
|
174
193
|
|
|
175
194
|
A workflow can also dispatch bitfab-replay.yml itself on any branch other than a
|
|
176
195
|
bitfab-replay/ snapshot branch, passing the replay as JSON in the request input:
|
|
@@ -190,6 +209,8 @@ the replay does. Traces that errored fail it only under --fail-on-error.
|
|
|
190
209
|
|
|
191
210
|
Follow a replay by the execution UUID it prints:
|
|
192
211
|
--cloud-watch ID | --cloud-status ID | --cloud-cancel ID | --cloud-cleanup ID
|
|
212
|
+
--cloud-cleanup removes the snapshot branch once the run completed, or when GitHub
|
|
213
|
+
shows no run for it, such as after a dispatch that failed.
|
|
193
214
|
|
|
194
215
|
Set up once:
|
|
195
216
|
--cloud-init [--secret NAME ...] [--environment NAME] [--run COMMAND]
|
|
@@ -211,6 +232,12 @@ class CommandError(RuntimeError):
|
|
|
211
232
|
self.http_status = http_status
|
|
212
233
|
|
|
213
234
|
|
|
235
|
+
class DispatchRejected(ValueError):
|
|
236
|
+
def __init__(self, message, record):
|
|
237
|
+
super().__init__(message)
|
|
238
|
+
self.record = record
|
|
239
|
+
|
|
240
|
+
|
|
214
241
|
class KeepRedirect(urllib.request.HTTPRedirectHandler):
|
|
215
242
|
def redirect_request(self, *args, **kwargs):
|
|
216
243
|
return None
|
|
@@ -609,7 +636,7 @@ def split_arguments(argv):
|
|
|
609
636
|
name = option_name(token)
|
|
610
637
|
if name in RENAMED_FLAGS:
|
|
611
638
|
raise ValueError(f"{name} is gone; use {RENAMED_FLAGS[name]}")
|
|
612
|
-
if name
|
|
639
|
+
if name in REPEATED_FLAGS:
|
|
613
640
|
if "=" in token:
|
|
614
641
|
value = token.split("=", 1)[1]
|
|
615
642
|
index += 1
|
|
@@ -617,8 +644,12 @@ def split_arguments(argv):
|
|
|
617
644
|
value = argv[index + 1]
|
|
618
645
|
index += 2
|
|
619
646
|
else:
|
|
620
|
-
raise ValueError(
|
|
621
|
-
|
|
647
|
+
raise ValueError(
|
|
648
|
+
f"{ENV_FLAG} needs NAME=VALUE or NAME"
|
|
649
|
+
if name == ENV_FLAG
|
|
650
|
+
else f"{name} needs a path"
|
|
651
|
+
)
|
|
652
|
+
cloud.setdefault(name, []).append(value)
|
|
622
653
|
elif name in CLOUD_VALUE_FLAGS:
|
|
623
654
|
if "=" in token:
|
|
624
655
|
value = token.split("=", 1)[1]
|
|
@@ -655,10 +686,6 @@ def parse(argv):
|
|
|
655
686
|
raise ValueError(
|
|
656
687
|
"Add --cloud to the replay command you run locally, such as bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud"
|
|
657
688
|
)
|
|
658
|
-
if not any(option_name(token) in SELECTION_FLAGS for token in replay):
|
|
659
|
-
raise ValueError(
|
|
660
|
-
"Select traces with --trace-ids, --dataset-ids, or --resume"
|
|
661
|
-
)
|
|
662
689
|
timeout = cloud.get("--cloud-timeout")
|
|
663
690
|
if timeout is not None and not (
|
|
664
691
|
timeout.isdigit() and 1 <= int(timeout) <= 7200
|
|
@@ -676,6 +703,10 @@ def parse(argv):
|
|
|
676
703
|
)
|
|
677
704
|
if ENV_FLAG in cloud:
|
|
678
705
|
cloud[ENV_FLAG] = replay_environment(cloud[ENV_FLAG])
|
|
706
|
+
if ALLOW_FLAG in cloud and "--cloud-ref" in cloud:
|
|
707
|
+
raise ValueError(
|
|
708
|
+
f"{ALLOW_FLAG} lets a file into the snapshot of your working tree, and --cloud-ref replays the branch on GitHub with no snapshot"
|
|
709
|
+
)
|
|
679
710
|
operation = "submit"
|
|
680
711
|
execution_id = cloud.get("--cloud-request-id") or str(uuid.uuid4())
|
|
681
712
|
if not UUID.fullmatch(execution_id):
|
|
@@ -761,6 +792,38 @@ def dry_run(args):
|
|
|
761
792
|
return "--dry-run" in args
|
|
762
793
|
|
|
763
794
|
|
|
795
|
+
def replay_arguments(argv):
|
|
796
|
+
try:
|
|
797
|
+
parsed = parse(argv)
|
|
798
|
+
except ValueError:
|
|
799
|
+
return None
|
|
800
|
+
return parsed["replay"] if parsed["operation"] == "submit" else None
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def check_arguments(replay):
|
|
804
|
+
if os.environ.get(ARGUMENTS_CHECKED_ENV) == "1":
|
|
805
|
+
return
|
|
806
|
+
try:
|
|
807
|
+
result = subprocess.run(
|
|
808
|
+
[*replay_command(), CHECK_ARGUMENTS_FLAG, *replay],
|
|
809
|
+
text=True,
|
|
810
|
+
capture_output=True,
|
|
811
|
+
stdin=subprocess.DEVNULL,
|
|
812
|
+
check=False,
|
|
813
|
+
timeout=CHECK_ARGUMENTS_SECONDS,
|
|
814
|
+
)
|
|
815
|
+
except subprocess.TimeoutExpired:
|
|
816
|
+
raise ValueError(
|
|
817
|
+
f"The replay command took more than {CHECK_ARGUMENTS_SECONDS} seconds to check these options, so nothing was uploaded; make sure it reaches the Bitfab replay entry point without waiting on input or a slow setup step"
|
|
818
|
+
) from None
|
|
819
|
+
if result.returncode:
|
|
820
|
+
message = (result.stderr or result.stdout).strip()
|
|
821
|
+
raise ValueError(
|
|
822
|
+
message
|
|
823
|
+
or f"The replay command refused these options (exit {result.returncode})"
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
|
|
764
827
|
def replay_request(root, parsed):
|
|
765
828
|
cwd = Path.cwd().resolve()
|
|
766
829
|
args, files = [], []
|
|
@@ -826,7 +889,7 @@ def sensitive(path):
|
|
|
826
889
|
)
|
|
827
890
|
|
|
828
891
|
|
|
829
|
-
def snapshot(root, workflow, files, request, *, preview=False):
|
|
892
|
+
def snapshot(root, workflow, files, request, *, preview=False, allowed=()):
|
|
830
893
|
if git(root, "ls-files", "-u"):
|
|
831
894
|
raise ValueError("Resolve merge conflicts before snapshotting")
|
|
832
895
|
head = git(root, "rev-parse", "HEAD")
|
|
@@ -843,10 +906,6 @@ def snapshot(root, workflow, files, request, *, preview=False):
|
|
|
843
906
|
raise ValueError(
|
|
844
907
|
f"{path} is a submodule or nested repository, which cloud snapshots do not support"
|
|
845
908
|
)
|
|
846
|
-
if sensitive(path):
|
|
847
|
-
raise ValueError(
|
|
848
|
-
f"Refusing credential-like file {path}; add it to .gitignore, or git rm --cached it when it is tracked"
|
|
849
|
-
)
|
|
850
909
|
ignored = [
|
|
851
910
|
path
|
|
852
911
|
for path in [f"{WORKFLOW_DIRECTORY}/{workflow}", *files]
|
|
@@ -867,6 +926,23 @@ def snapshot(root, workflow, files, request, *, preview=False):
|
|
|
867
926
|
]
|
|
868
927
|
new_files = sorted(path for status, path in changes if status == "A")
|
|
869
928
|
changed_files = sorted(path for status, path in changes if status != "A")
|
|
929
|
+
unpushed = git(
|
|
930
|
+
root,
|
|
931
|
+
"log",
|
|
932
|
+
"--format=",
|
|
933
|
+
"--name-only",
|
|
934
|
+
"--no-renames",
|
|
935
|
+
"--diff-filter=AM",
|
|
936
|
+
head,
|
|
937
|
+
"--not",
|
|
938
|
+
"--remotes",
|
|
939
|
+
).splitlines()
|
|
940
|
+
uploaded = ({*new_files, *changed_files} & tracked) | set(unpushed)
|
|
941
|
+
for path in sorted(uploaded - set(allowed)):
|
|
942
|
+
if sensitive(path):
|
|
943
|
+
raise ValueError(
|
|
944
|
+
f"Refusing to upload {path}, which looks like a credential and is not on GitHub yet; add it to .gitignore and take it out of any commit you have not pushed, or pass {ALLOW_FLAG} {path} if it is safe to upload. If it is already on GitHub, run git fetch so this check can see it"
|
|
945
|
+
)
|
|
870
946
|
sizes = {path: (root / path).lstat().st_size for path in new_files}
|
|
871
947
|
size = sum(sizes.values())
|
|
872
948
|
if size > NEW_FILES_LIMIT:
|
|
@@ -943,22 +1019,24 @@ def find_run(record):
|
|
|
943
1019
|
|
|
944
1020
|
|
|
945
1021
|
def find_branch_run(record):
|
|
1022
|
+
dispatched = calendar.timegm(time.strptime(record["dispatchedAt"], TIMESTAMP))
|
|
946
1023
|
query = urlencode(
|
|
947
1024
|
{
|
|
948
1025
|
"event": "workflow_dispatch",
|
|
949
1026
|
"branch": record["branch"],
|
|
950
|
-
"created": ">=" +
|
|
1027
|
+
"created": ">=" + timestamp(dispatched - CLOCK_SKEW_SECONDS),
|
|
951
1028
|
"per_page": 100,
|
|
952
1029
|
}
|
|
953
1030
|
)
|
|
954
1031
|
runs = api(
|
|
955
1032
|
record["repository"], f"actions/workflows/{record['workflow']}/runs?{query}"
|
|
956
1033
|
)
|
|
957
|
-
|
|
958
|
-
run
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1034
|
+
on_branch = [
|
|
1035
|
+
run for run in runs["workflow_runs"] if run["head_branch"] == record["branch"]
|
|
1036
|
+
]
|
|
1037
|
+
named = [run for run in on_branch if record["id"] in run.get("display_title", "")]
|
|
1038
|
+
matches = named or [
|
|
1039
|
+
run for run in on_branch if run.get("created_at", "") >= record["dispatchedAt"]
|
|
962
1040
|
]
|
|
963
1041
|
if len(matches) > 1:
|
|
964
1042
|
raise ValueError(
|
|
@@ -967,8 +1045,12 @@ def find_branch_run(record):
|
|
|
967
1045
|
return matches[0] if matches else None
|
|
968
1046
|
|
|
969
1047
|
|
|
1048
|
+
def timestamp(seconds):
|
|
1049
|
+
return time.strftime(TIMESTAMP, time.gmtime(seconds))
|
|
1050
|
+
|
|
1051
|
+
|
|
970
1052
|
def status(record, *, fetch_result=True):
|
|
971
|
-
if record["state"]
|
|
1053
|
+
if record["state"] in ("prepared", "dispatch_rejected"):
|
|
972
1054
|
return record
|
|
973
1055
|
run = (
|
|
974
1056
|
api(record["repository"], f"actions/runs/{record['runId']}")
|
|
@@ -1091,7 +1173,7 @@ def print_outcome(found):
|
|
|
1091
1173
|
if summary.get("experimentUrl")
|
|
1092
1174
|
else ""
|
|
1093
1175
|
)
|
|
1094
|
-
+ ". Add --cloud-full-output to print every item here instead"
|
|
1176
|
+
+ "; the Bitfab MCP tools read them. Add --cloud-full-output to print every item here instead"
|
|
1095
1177
|
)
|
|
1096
1178
|
if record.get("stoppedEarly"):
|
|
1097
1179
|
progress(
|
|
@@ -1112,7 +1194,7 @@ def cleanup(root, record):
|
|
|
1112
1194
|
if record.get("snapshot") is False:
|
|
1113
1195
|
record["cleaned"] = True
|
|
1114
1196
|
return
|
|
1115
|
-
if record.get("state") not in
|
|
1197
|
+
if record.get("state") not in CLEANABLE_STATES:
|
|
1116
1198
|
raise ValueError(
|
|
1117
1199
|
"Cleanup requires a confirmed completed GitHub run; cancel and wait first"
|
|
1118
1200
|
)
|
|
@@ -1140,7 +1222,7 @@ def http_status(error):
|
|
|
1140
1222
|
return f" (HTTP {status})" if status else ""
|
|
1141
1223
|
|
|
1142
1224
|
|
|
1143
|
-
def preflight(repo, workflow, *, pushes=True):
|
|
1225
|
+
def preflight(repo, workflow, *, pushes=True, full_output=False, allows_files=False):
|
|
1144
1226
|
access = github_access()
|
|
1145
1227
|
try:
|
|
1146
1228
|
details = api(repo, "")
|
|
@@ -1152,6 +1234,14 @@ def preflight(repo, workflow, *, pushes=True):
|
|
|
1152
1234
|
raise ValueError(
|
|
1153
1235
|
f"GitHub access from {access['source']} cannot push to {repo}; use an account or token with write access to it"
|
|
1154
1236
|
)
|
|
1237
|
+
if full_output and (details or {}).get("private") is False:
|
|
1238
|
+
raise ValueError(
|
|
1239
|
+
f"--cloud-full-output writes every trace's input and output into the Actions job log, and anyone can read the logs of {repo} because it is public; leave it off and read the items in the experiment with the Bitfab MCP tools"
|
|
1240
|
+
)
|
|
1241
|
+
if allows_files and (details or {}).get("private") is False:
|
|
1242
|
+
raise ValueError(
|
|
1243
|
+
f"{ALLOW_FLAG} uploads a credential-like file to a snapshot branch, and anyone can read the branches of {repo} because it is public"
|
|
1244
|
+
)
|
|
1155
1245
|
path = f"{WORKFLOW_DIRECTORY}/{workflow}"
|
|
1156
1246
|
try:
|
|
1157
1247
|
registered = api(repo, f"actions/workflows/{workflow}")
|
|
@@ -1177,14 +1267,24 @@ def describe_snapshot(source):
|
|
|
1177
1267
|
)
|
|
1178
1268
|
|
|
1179
1269
|
|
|
1270
|
+
def allowed_files(root, parsed):
|
|
1271
|
+
cwd = Path.cwd().resolve()
|
|
1272
|
+
return [
|
|
1273
|
+
repository_path(root, cwd / value)
|
|
1274
|
+
for value in parsed["cloud"].get(ALLOW_FLAG, [])
|
|
1275
|
+
]
|
|
1276
|
+
|
|
1277
|
+
|
|
1180
1278
|
def submit(root, repo, workflow, parsed, path):
|
|
1181
1279
|
execution_id = parsed["id"]
|
|
1182
1280
|
request, files = replay_request(root, parsed)
|
|
1281
|
+
check_arguments(parsed["replay"])
|
|
1282
|
+
allowed = allowed_files(root, parsed)
|
|
1183
1283
|
if "--cloud-preview" in parsed["switches"]:
|
|
1184
1284
|
return {
|
|
1185
1285
|
"preview": True,
|
|
1186
1286
|
"repository": repo,
|
|
1187
|
-
**snapshot(root, workflow, files, request, preview=True),
|
|
1287
|
+
**snapshot(root, workflow, files, request, preview=True, allowed=allowed),
|
|
1188
1288
|
}
|
|
1189
1289
|
if path.exists():
|
|
1190
1290
|
record = json.loads(path.read_text())
|
|
@@ -1195,13 +1295,19 @@ def submit(root, repo, workflow, parsed, path):
|
|
|
1195
1295
|
raise ValueError(
|
|
1196
1296
|
"Submission stopped before dispatch. Use --cloud-cleanup, then submit a new execution UUID"
|
|
1197
1297
|
)
|
|
1298
|
+
refuse_rejected(record)
|
|
1198
1299
|
return record
|
|
1199
|
-
preflight(
|
|
1300
|
+
preflight(
|
|
1301
|
+
repo,
|
|
1302
|
+
workflow,
|
|
1303
|
+
full_output=request.get("fullOutput") is True,
|
|
1304
|
+
allows_files=bool(allowed),
|
|
1305
|
+
)
|
|
1200
1306
|
if dry_run(request["args"]):
|
|
1201
1307
|
progress(
|
|
1202
1308
|
"--dry-run dispatches a GitHub run that checks secrets and resolves the traces without replaying; it receives the workflow's secrets"
|
|
1203
1309
|
)
|
|
1204
|
-
source = snapshot(root, workflow, files, request)
|
|
1310
|
+
source = snapshot(root, workflow, files, request, allowed=allowed)
|
|
1205
1311
|
describe_snapshot(source)
|
|
1206
1312
|
record = {
|
|
1207
1313
|
"id": execution_id,
|
|
@@ -1264,8 +1370,22 @@ def dispatch(record, inputs, declared, *, run_details=False):
|
|
|
1264
1370
|
raise
|
|
1265
1371
|
response = api(record["repository"], path, method="POST", payload=payload)
|
|
1266
1372
|
except RuntimeError as error:
|
|
1373
|
+
code = getattr(error, "http_status", None)
|
|
1374
|
+
problem = f"Dispatching {record['workflow']} on {record['branch']} failed{http_status(error)}"
|
|
1375
|
+
if code is not None and 400 <= code < 500 and code != 429:
|
|
1376
|
+
record["state"] = "dispatch_rejected"
|
|
1377
|
+
raise DispatchRejected(
|
|
1378
|
+
f"{problem}, so no run started"
|
|
1379
|
+
+ (
|
|
1380
|
+
""
|
|
1381
|
+
if record.get("snapshot") is False
|
|
1382
|
+
else " and the snapshot branch was removed"
|
|
1383
|
+
)
|
|
1384
|
+
+ ". The GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch",
|
|
1385
|
+
record,
|
|
1386
|
+
) from error
|
|
1267
1387
|
raise ValueError(
|
|
1268
|
-
f"
|
|
1388
|
+
f"{problem}, and GitHub may still have started it. Follow it with --cloud-watch {record['id']}, or once --cloud-status {record['id']} shows no run, remove it with --cloud-cleanup {record['id']}"
|
|
1269
1389
|
) from error
|
|
1270
1390
|
if isinstance(response, dict) and response.get("workflow_run_id"):
|
|
1271
1391
|
record["runId"] = response["workflow_run_id"]
|
|
@@ -1302,6 +1422,7 @@ def submit_as_is(root, repo, workflow, parsed, path):
|
|
|
1302
1422
|
execution_id = parsed["id"]
|
|
1303
1423
|
branch = parsed["cloud"]["--cloud-ref"]
|
|
1304
1424
|
request, _ = replay_request(root, parsed)
|
|
1425
|
+
check_arguments(parsed["replay"])
|
|
1305
1426
|
sha = branch_head(repo, branch)
|
|
1306
1427
|
if "--cloud-preview" in parsed["switches"]:
|
|
1307
1428
|
return {
|
|
@@ -1320,8 +1441,11 @@ def submit_as_is(root, repo, workflow, parsed, path):
|
|
|
1320
1441
|
):
|
|
1321
1442
|
raise ValueError("Execution ID already belongs to a different request")
|
|
1322
1443
|
status(record, fetch_result=False)
|
|
1444
|
+
refuse_rejected(record)
|
|
1323
1445
|
return record
|
|
1324
|
-
preflight(
|
|
1446
|
+
preflight(
|
|
1447
|
+
repo, workflow, pushes=False, full_output=request.get("fullOutput") is True
|
|
1448
|
+
)
|
|
1325
1449
|
declared = declared_inputs(branch_workflow(repo, workflow, sha))
|
|
1326
1450
|
if "request" not in declared:
|
|
1327
1451
|
raise ValueError(
|
|
@@ -1336,9 +1460,7 @@ def submit_as_is(root, repo, workflow, parsed, path):
|
|
|
1336
1460
|
"request": request,
|
|
1337
1461
|
"state": "dispatch_unknown",
|
|
1338
1462
|
"sha": sha,
|
|
1339
|
-
"dispatchedAt": time.
|
|
1340
|
-
"%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 5)
|
|
1341
|
-
),
|
|
1463
|
+
"dispatchedAt": timestamp(time.time() - 5),
|
|
1342
1464
|
}
|
|
1343
1465
|
save(path, record)
|
|
1344
1466
|
progress(
|
|
@@ -1363,12 +1485,20 @@ def report_steps(record, shown):
|
|
|
1363
1485
|
progress(step.get("name"))
|
|
1364
1486
|
|
|
1365
1487
|
|
|
1488
|
+
def refuse_rejected(record):
|
|
1489
|
+
if record["state"] == "dispatch_rejected":
|
|
1490
|
+
raise ValueError(
|
|
1491
|
+
"GitHub refused this execution's dispatch, so no run exists; submit the replay again without --cloud-request-id"
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
|
|
1366
1495
|
def wait(record, path):
|
|
1367
1496
|
started = time.monotonic()
|
|
1368
1497
|
announced, steps = None, set()
|
|
1369
1498
|
while record["state"] != "completed":
|
|
1370
1499
|
status(record, fetch_result=False)
|
|
1371
1500
|
save(path, record)
|
|
1501
|
+
refuse_rejected(record)
|
|
1372
1502
|
if record["state"] != announced and record.get("url"):
|
|
1373
1503
|
announced = record["state"]
|
|
1374
1504
|
progress(f"{announced.replace('_', ' ').capitalize()}: {record['url']}")
|
|
@@ -1398,7 +1528,12 @@ def run_cli(argv):
|
|
|
1398
1528
|
with execution_lock(directory, execution_id):
|
|
1399
1529
|
if operation == "submit":
|
|
1400
1530
|
submitter = submit_as_is if "--cloud-ref" in parsed["cloud"] else submit
|
|
1401
|
-
|
|
1531
|
+
try:
|
|
1532
|
+
record = submitter(root, repo, workflow, parsed, path)
|
|
1533
|
+
except DispatchRejected as error:
|
|
1534
|
+
cleanup(root, error.record)
|
|
1535
|
+
save(path, error.record)
|
|
1536
|
+
raise
|
|
1402
1537
|
if record.get("preview"):
|
|
1403
1538
|
return record, None
|
|
1404
1539
|
else:
|
|
@@ -1426,11 +1561,13 @@ def run_cli(argv):
|
|
|
1426
1561
|
wait(record, path)
|
|
1427
1562
|
found = outcome(record)
|
|
1428
1563
|
if record["state"] == "completed" or (
|
|
1429
|
-
operation == "cleanup" and record["state"]
|
|
1564
|
+
operation == "cleanup" and record["state"] in CLEANABLE_STATES
|
|
1430
1565
|
):
|
|
1431
1566
|
cleanup(root, record)
|
|
1432
1567
|
elif operation == "cleanup":
|
|
1433
|
-
raise ValueError(
|
|
1568
|
+
raise ValueError(
|
|
1569
|
+
f"The GitHub run is {record['state'].replace('_', ' ')}; cancel it with --cloud-cancel {record['id']} and clean up once it completes. No branch was deleted"
|
|
1570
|
+
)
|
|
1434
1571
|
save(path, record)
|
|
1435
1572
|
return record, found
|
|
1436
1573
|
|
|
@@ -1922,6 +2059,8 @@ def run_command(
|
|
|
1922
2059
|
stop(child)
|
|
1923
2060
|
raise
|
|
1924
2061
|
finally:
|
|
2062
|
+
with contextlib.suppress(ProcessLookupError, PermissionError):
|
|
2063
|
+
os.killpg(child.pid, signal.SIGKILL)
|
|
1925
2064
|
reader.join(timeout=5)
|
|
1926
2065
|
code = child.returncode if child.returncode >= 0 else 1
|
|
1927
2066
|
text, omitted = relayed_stdout(output, code, summarize)
|
|
@@ -1938,6 +2077,9 @@ def stop(child):
|
|
|
1938
2077
|
child.wait()
|
|
1939
2078
|
|
|
1940
2079
|
|
|
2080
|
+
PNPM_FOR_LOCKFILE = {"9": "10", "6": "8", "5": "7"}
|
|
2081
|
+
DEFAULT_PNPM = "10"
|
|
2082
|
+
RUBY_VERSION_FILES = [".ruby-version", ".tool-versions"]
|
|
1941
2083
|
PLAIN_SCALAR = re.compile(r"[A-Za-z0-9_./][A-Za-z0-9_ ./@*+=,()-]*")
|
|
1942
2084
|
YAML_WORDS = {"true", "false", "yes", "no", "on", "off", "null", "~", "y", "n"}
|
|
1943
2085
|
|
|
@@ -2048,9 +2190,16 @@ def typescript_project(root, start):
|
|
|
2048
2190
|
}[manager]
|
|
2049
2191
|
steps = []
|
|
2050
2192
|
if manager == "pnpm":
|
|
2051
|
-
steps.append(
|
|
2193
|
+
steps.append(pnpm_setup(root, directory, lockfile))
|
|
2052
2194
|
if manager == "bun":
|
|
2053
2195
|
steps.append({"uses": ACTIONS["bun"]})
|
|
2196
|
+
cache = {}
|
|
2197
|
+
if manager in ("pnpm", "yarn", "npm") and lockfile:
|
|
2198
|
+
cache["cache"] = manager
|
|
2199
|
+
if directory != root:
|
|
2200
|
+
cache["cache-dependency-path"] = (
|
|
2201
|
+
(directory / lockfile).relative_to(root).as_posix()
|
|
2202
|
+
)
|
|
2054
2203
|
steps += [
|
|
2055
2204
|
{
|
|
2056
2205
|
"uses": ACTIONS["node"],
|
|
@@ -2058,11 +2207,7 @@ def typescript_project(root, start):
|
|
|
2058
2207
|
**version_input(
|
|
2059
2208
|
root, start, "node", [".nvmrc", ".node-version"], "lts/*"
|
|
2060
2209
|
),
|
|
2061
|
-
**
|
|
2062
|
-
{"cache": manager}
|
|
2063
|
-
if manager in ("pnpm", "yarn", "npm") and lockfile
|
|
2064
|
-
else {}
|
|
2065
|
-
),
|
|
2210
|
+
**cache,
|
|
2066
2211
|
},
|
|
2067
2212
|
},
|
|
2068
2213
|
{**step_directory(root, directory or start), "run": install},
|
|
@@ -2076,6 +2221,28 @@ def typescript_project(root, start):
|
|
|
2076
2221
|
return steps, run
|
|
2077
2222
|
|
|
2078
2223
|
|
|
2224
|
+
def pnpm_setup(root, directory, lockfile):
|
|
2225
|
+
package = directory / "package.json"
|
|
2226
|
+
try:
|
|
2227
|
+
declared = json.loads(package.read_text()).get("packageManager", "")
|
|
2228
|
+
except (OSError, ValueError, AttributeError):
|
|
2229
|
+
declared = ""
|
|
2230
|
+
if isinstance(declared, str) and declared.startswith("pnpm@"):
|
|
2231
|
+
if directory == root:
|
|
2232
|
+
return {"uses": ACTIONS["pnpm"]}
|
|
2233
|
+
return {
|
|
2234
|
+
"uses": ACTIONS["pnpm"],
|
|
2235
|
+
"with": {"package_json_file": package.relative_to(root).as_posix()},
|
|
2236
|
+
}
|
|
2237
|
+
found = re.search(
|
|
2238
|
+
r"^lockfileVersion:\s*['\"]?(\d+)",
|
|
2239
|
+
(directory / lockfile).read_text(errors="replace"),
|
|
2240
|
+
re.MULTILINE,
|
|
2241
|
+
)
|
|
2242
|
+
version = PNPM_FOR_LOCKFILE.get(found[1] if found else "", DEFAULT_PNPM)
|
|
2243
|
+
return {"uses": ACTIONS["pnpm"], "with": {"version": version}}
|
|
2244
|
+
|
|
2245
|
+
|
|
2079
2246
|
def python_project(root, start):
|
|
2080
2247
|
directory, lockfile = find_upward(
|
|
2081
2248
|
root, start, ["uv.lock", "poetry.lock", "requirements.txt", "pyproject.toml"]
|
|
@@ -2108,12 +2275,33 @@ def python_project(root, start):
|
|
|
2108
2275
|
|
|
2109
2276
|
def ruby_project(root, start):
|
|
2110
2277
|
directory, _ = find_upward(root, start, ["Gemfile"])
|
|
2111
|
-
|
|
2278
|
+
directory = directory or start
|
|
2279
|
+
found, name = find_upward(root, directory, RUBY_VERSION_FILES)
|
|
2280
|
+
version = (
|
|
2281
|
+
{}
|
|
2282
|
+
if found == directory
|
|
2283
|
+
else {"ruby-version": (name and ruby_version(found / name)) or "ruby"}
|
|
2284
|
+
)
|
|
2112
2285
|
return [
|
|
2113
|
-
{
|
|
2286
|
+
{
|
|
2287
|
+
"uses": ACTIONS["ruby"],
|
|
2288
|
+
"with": {
|
|
2289
|
+
**version,
|
|
2290
|
+
"bundler-cache": True,
|
|
2291
|
+
**step_directory(root, directory),
|
|
2292
|
+
},
|
|
2293
|
+
}
|
|
2114
2294
|
], "bundle exec bitfab-replay"
|
|
2115
2295
|
|
|
2116
2296
|
|
|
2297
|
+
def ruby_version(path):
|
|
2298
|
+
text = path.read_text(errors="replace")
|
|
2299
|
+
if path.name == ".tool-versions":
|
|
2300
|
+
found = re.search(r"^ruby\s+(\S+)", text, re.MULTILINE)
|
|
2301
|
+
return found[1] if found else None
|
|
2302
|
+
return text.strip() or None
|
|
2303
|
+
|
|
2304
|
+
|
|
2117
2305
|
def go_project(root, start):
|
|
2118
2306
|
directory, name = find_upward(root, start, ["go.mod"])
|
|
2119
2307
|
version = (
|
|
@@ -2188,9 +2376,13 @@ def workflow_document(steps, run, directory, env, *, runs_on, environment):
|
|
|
2188
2376
|
]
|
|
2189
2377
|
return {
|
|
2190
2378
|
"name": "Bitfab replay",
|
|
2379
|
+
"run-name": "Bitfab replay ${{ inputs.execution_id || github.ref_name }}",
|
|
2191
2380
|
"on": {
|
|
2192
2381
|
"workflow_dispatch": {
|
|
2193
|
-
"inputs": {
|
|
2382
|
+
"inputs": {
|
|
2383
|
+
"execution_id": {"type": "string", "required": False},
|
|
2384
|
+
"request": {"type": "string", "required": False},
|
|
2385
|
+
}
|
|
2194
2386
|
}
|
|
2195
2387
|
},
|
|
2196
2388
|
"permissions": {"contents": "read"},
|
|
@@ -2320,6 +2512,9 @@ def main():
|
|
|
2320
2512
|
if argv[:1] == ["--cloud-secrets"]:
|
|
2321
2513
|
print_json(configure_secrets(argv[1:]))
|
|
2322
2514
|
return 0
|
|
2515
|
+
if argv[:1] == [REPLAY_ARGUMENTS_FLAG]:
|
|
2516
|
+
print_json(replay_arguments(argv[1:]))
|
|
2517
|
+
return 0
|
|
2323
2518
|
if argv == ["--cloud-execute"]:
|
|
2324
2519
|
result = execute()
|
|
2325
2520
|
annotate(result)
|
|
@@ -2334,7 +2529,7 @@ def main():
|
|
|
2334
2529
|
return record.get("exitCode", 0) if record.get("state") == "completed" else 0
|
|
2335
2530
|
except KeyboardInterrupt:
|
|
2336
2531
|
progress(
|
|
2337
|
-
"Detached.
|
|
2532
|
+
"Detached. A GitHub run that already started continues; follow it with --cloud-watch and the printed execution UUID, or stop it with --cloud-cancel and remove its snapshot branch with --cloud-cleanup once it completes"
|
|
2338
2533
|
)
|
|
2339
2534
|
return 130
|
|
2340
2535
|
except (
|