bitfab 0.63.3 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/cloudReplay.py +663 -757
- data/lib/bitfab/cloud_replay_cli.rb +10 -43
- data/lib/bitfab/replay_cli.rb +2 -2
- data/lib/bitfab/version.rb +1 -1
- metadata +1 -1
data/lib/bitfab/cloudReplay.py
CHANGED
|
@@ -19,6 +19,7 @@ import time
|
|
|
19
19
|
import urllib.error
|
|
20
20
|
import urllib.request
|
|
21
21
|
import uuid
|
|
22
|
+
import zlib
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
from urllib.parse import urlencode
|
|
24
25
|
|
|
@@ -26,19 +27,21 @@ WORKFLOW = "bitfab-replay.yml"
|
|
|
26
27
|
WORKFLOW_NAMES = (WORKFLOW, "bitfab-replay.yaml")
|
|
27
28
|
WORKFLOW_DIRECTORY = ".github/workflows"
|
|
28
29
|
OLD_CONFIG = ".bitfab/cloud.json"
|
|
29
|
-
OLD_SCRIPT = ".bitfab/cloudReplay.py"
|
|
30
|
-
OLD_RUNNER_ENV = ("BITFAB_CLOUD_REQUEST", "BITFAB_EXECUTION_ID", "BITFAB_COMMIT_SHA")
|
|
31
|
-
OLD_UPLOAD_STEP = "Save replay identity"
|
|
32
|
-
OLD_JOB_TIMEOUT = 35
|
|
33
30
|
PREFIX = "bitfab-replay/"
|
|
34
31
|
DEFAULT_SECRET_PREFIX = "BITFAB_CLOUD_"
|
|
35
32
|
API_VERSION = "2026-03-10"
|
|
36
|
-
RUN_NAME = "Bitfab replay ${{ inputs.execution_id }}"
|
|
37
|
-
RESULT_TITLE = "Bitfab replay result"
|
|
38
33
|
REPLAY_COMMAND_ENV = "BITFAB_REPLAY_COMMAND"
|
|
39
34
|
SDK_LANGUAGE_ENV = "BITFAB_SDK_LANGUAGE"
|
|
40
35
|
CHECK_COMMAND_ENV = "BITFAB_REPLAY_CHECK"
|
|
41
|
-
REQUEST_VERSION =
|
|
36
|
+
REQUEST_VERSION = 3
|
|
37
|
+
RESULT_LINE = "bitfab-replay-result "
|
|
38
|
+
RESULT_CHUNK = 4000
|
|
39
|
+
OUTPUT_BEGIN = "bitfab-replay-output-begin"
|
|
40
|
+
OUTPUT_END = "bitfab-replay-output-end"
|
|
41
|
+
LOG_TAIL_LINES = 40
|
|
42
|
+
OUTPUT_LIMIT = 16 * 1024 * 1024
|
|
43
|
+
NEW_FILES_LIMIT = 20 * 1024 * 1024
|
|
44
|
+
POLL_SECONDS = 5
|
|
42
45
|
ITEM_ERROR_FIELDS = (
|
|
43
46
|
"error",
|
|
44
47
|
"traceError",
|
|
@@ -49,10 +52,12 @@ ITEM_ERROR_FIELDS = (
|
|
|
49
52
|
ITEM_ID_FIELDS = ("originalTraceId", "original_trace_id", "traceId", "trace_id")
|
|
50
53
|
ITEM_ERROR_LINES = 20
|
|
51
54
|
ITEM_ERROR_LENGTH = 500
|
|
52
|
-
OUTPUT_TAIL_LENGTH = 4000
|
|
53
55
|
UUID = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$")
|
|
54
56
|
SHA = re.compile(r"^[0-9a-f]{40}$")
|
|
55
|
-
|
|
57
|
+
GITHUB_REMOTE = re.compile(
|
|
58
|
+
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?"
|
|
59
|
+
)
|
|
60
|
+
LOG_TIMESTAMP = re.compile(r"^?\d{4}-\d\d-\d\dT[\d:.]+Z ?")
|
|
56
61
|
EXPERIMENT_LINE = re.compile(rb"^\[replay\] Experiment ([0-9a-f-]{36}):")
|
|
57
62
|
SECRET_REFERENCE = re.compile(
|
|
58
63
|
r"[\"']?([A-Za-z_][A-Za-z0-9_]*)[\"']?[ \t]*:[ \t]*[\"']?\$\{\{\s*secrets(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[\s*[\"']([A-Za-z_][A-Za-z0-9_]*)[\"']\s*\])\s*\}\}"
|
|
@@ -64,27 +69,20 @@ ENVIRONMENT_SETTING = re.compile(
|
|
|
64
69
|
PUSH_TRIGGER = re.compile(
|
|
65
70
|
r"^[ \t]*\"?(?:on\"?[ \t]*:.*\bpush\b|push\"?[ \t]*:)", re.MULTILINE
|
|
66
71
|
)
|
|
67
|
-
|
|
72
|
+
DECLARED_INPUTS = re.compile(r"^[ \t]*\"?execution_id\"?[ \t]*:", re.MULTILINE)
|
|
73
|
+
FOLLOW_FLAGS = (
|
|
68
74
|
"--cloud-status",
|
|
69
75
|
"--cloud-watch",
|
|
70
76
|
"--cloud-cancel",
|
|
71
77
|
"--cloud-cleanup",
|
|
72
78
|
)
|
|
73
|
-
CLOUD_VALUE_FLAGS = (
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
"--cloud-
|
|
77
|
-
"--cloud-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
"--cloud",
|
|
81
|
-
"--cloud-dry-run",
|
|
82
|
-
"--cloud-detach",
|
|
83
|
-
"--cloud-check",
|
|
84
|
-
"--fail-on-error",
|
|
85
|
-
"--dry-run",
|
|
86
|
-
)
|
|
87
|
-
CHECK_SWITCHES = ("--cloud-check", "--dry-run")
|
|
79
|
+
CLOUD_VALUE_FLAGS = (*FOLLOW_FLAGS, "--cloud-request-id", "--cloud-timeout")
|
|
80
|
+
CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach")
|
|
81
|
+
RENAMED_FLAGS = {
|
|
82
|
+
"--cloud-dry-run": "--cloud-preview",
|
|
83
|
+
"--cloud-check": "--dry-run",
|
|
84
|
+
"--cloud-include": "nothing: new files that git does not ignore are already in the snapshot",
|
|
85
|
+
}
|
|
88
86
|
SEED_FLAGS = ("--seed", "--cases", "--from-trace", "--run")
|
|
89
87
|
PATH_FLAGS = ("--registry", "--params", "--code-change")
|
|
90
88
|
CODE_CHANGE_FLAGS = ("--code-change", "--no-code-change")
|
|
@@ -109,36 +107,32 @@ HELP = """Cloud replay: run the replay you run locally on GitHub Actions by addi
|
|
|
109
107
|
|
|
110
108
|
bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud
|
|
111
109
|
|
|
112
|
-
Every replay option works as it does locally
|
|
113
|
-
The runner
|
|
114
|
-
|
|
115
|
-
|
|
110
|
+
Every replay option works as it does locally, including --dry-run and --fail-on-error.
|
|
111
|
+
The runner replays a snapshot of your working tree (changed files, plus new files git
|
|
112
|
+
does not ignore) from the same directory, and this command prints the replay's output
|
|
113
|
+
and exits with its exit code. Credential-like files are refused. Requires git and
|
|
114
|
+
Python 3.10+ on macOS or Linux, with a github.com origin.
|
|
116
115
|
|
|
117
116
|
Options added by --cloud:
|
|
118
|
-
--
|
|
119
|
-
maps has a value, BITFAB_REPLAY_CHECK passes when set, and the
|
|
120
|
-
traces resolve. --cloud-check is the same.
|
|
121
|
-
--cloud-dry-run Show what the snapshot would contain and push nothing.
|
|
122
|
-
--cloud-include PATH Add a new file to the snapshot (repeatable).
|
|
117
|
+
--cloud-preview List what the snapshot would contain and push nothing.
|
|
123
118
|
--cloud-detach Return after dispatch instead of waiting.
|
|
124
119
|
--cloud-timeout MIN Stop the replay after MIN minutes (1..7200).
|
|
125
120
|
--cloud-request-id ID Recover a submission whose response was lost.
|
|
126
|
-
--fail-on-error Exit 1 when any replayed item errored.
|
|
127
121
|
|
|
128
122
|
Follow a replay by the execution UUID it prints:
|
|
129
|
-
--cloud-
|
|
123
|
+
--cloud-watch ID | --cloud-status ID | --cloud-cancel ID | --cloud-cleanup ID
|
|
130
124
|
|
|
131
125
|
Set up once:
|
|
132
126
|
--cloud-init [--secret NAME ...] [--environment NAME] [--run COMMAND]
|
|
133
127
|
[--runs-on LABEL] [--secret-prefix PREFIX] [--check COMMAND]
|
|
134
128
|
Writes .github/workflows/bitfab-replay.yml, the only file cloud replay keeps
|
|
135
|
-
in the repository
|
|
129
|
+
in the repository, and lists what is left to do.
|
|
136
130
|
--cloud-secrets --env-file FILE [NAME ...] [--environment NAME] [--dry-run]
|
|
137
|
-
|
|
131
|
+
Copies local values into the GitHub secrets the workflow reads.
|
|
138
132
|
|
|
139
133
|
GitHub access comes from the GitHub CLI when it is logged in, otherwise from GH_TOKEN
|
|
140
|
-
or GITHUB_TOKEN, otherwise from the github.com credential git already stores.
|
|
141
|
-
|
|
134
|
+
or GITHUB_TOKEN, otherwise from the github.com credential git already stores. Without
|
|
135
|
+
the GitHub CLI, --cloud-secrets lists the secrets to create by hand and where.
|
|
142
136
|
"""
|
|
143
137
|
|
|
144
138
|
|
|
@@ -148,6 +142,11 @@ class CommandError(RuntimeError):
|
|
|
148
142
|
self.http_status = http_status
|
|
149
143
|
|
|
150
144
|
|
|
145
|
+
class KeepRedirect(urllib.request.HTTPRedirectHandler):
|
|
146
|
+
def redirect_request(self, *args, **kwargs):
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
151
150
|
def command(args, *, cwd=None, env=None, timeout=60, input=None):
|
|
152
151
|
result = subprocess.run(
|
|
153
152
|
args,
|
|
@@ -175,6 +174,10 @@ def git(root, *args, env=None, input=None):
|
|
|
175
174
|
return command(["git", "-C", str(root), *args], env=env, input=input)
|
|
176
175
|
|
|
177
176
|
|
|
177
|
+
def progress(message):
|
|
178
|
+
print(f"[cloud] {message}", file=sys.stderr, flush=True)
|
|
179
|
+
|
|
180
|
+
|
|
178
181
|
def root_directory():
|
|
179
182
|
return Path(command(["git", "rev-parse", "--show-toplevel"])).resolve()
|
|
180
183
|
|
|
@@ -195,12 +198,9 @@ def within(root, value):
|
|
|
195
198
|
|
|
196
199
|
def repository_path(root, path):
|
|
197
200
|
try:
|
|
198
|
-
|
|
201
|
+
return Path(path).resolve().relative_to(root).as_posix()
|
|
199
202
|
except ValueError as error:
|
|
200
203
|
raise ValueError(f"{path} must be inside the repository") from error
|
|
201
|
-
if sensitive(relative):
|
|
202
|
-
raise ValueError(f"Refusing credential-like file: {relative}")
|
|
203
|
-
return relative
|
|
204
204
|
|
|
205
205
|
|
|
206
206
|
def working_directory(root):
|
|
@@ -212,21 +212,15 @@ def working_directory(root):
|
|
|
212
212
|
|
|
213
213
|
|
|
214
214
|
def repository(root):
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?", remote
|
|
218
|
-
)
|
|
219
|
-
if match is None:
|
|
215
|
+
fetch = GITHUB_REMOTE.fullmatch(git(root, "remote", "get-url", "origin"))
|
|
216
|
+
if fetch is None:
|
|
220
217
|
raise ValueError(
|
|
221
218
|
"origin must be a github.com repository without embedded credentials"
|
|
222
219
|
)
|
|
223
|
-
push = git(root, "remote", "get-url", "--push", "origin")
|
|
224
|
-
|
|
225
|
-
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?", push
|
|
226
|
-
)
|
|
227
|
-
if other is None or other[1].lower() != match[1].lower():
|
|
220
|
+
push = GITHUB_REMOTE.fullmatch(git(root, "remote", "get-url", "--push", "origin"))
|
|
221
|
+
if push is None or push[1].lower() != fetch[1].lower():
|
|
228
222
|
raise ValueError("origin fetch and push repositories must match")
|
|
229
|
-
return
|
|
223
|
+
return fetch[1]
|
|
230
224
|
|
|
231
225
|
|
|
232
226
|
def find_workflow(root):
|
|
@@ -236,11 +230,15 @@ def find_workflow(root):
|
|
|
236
230
|
return None
|
|
237
231
|
|
|
238
232
|
|
|
239
|
-
def
|
|
233
|
+
def refuse_old_setup(root):
|
|
240
234
|
if within(root, OLD_CONFIG).exists():
|
|
241
235
|
raise ValueError(
|
|
242
|
-
f"This cloud replay setup was made by an older SDK.
|
|
236
|
+
f"This cloud replay setup was made by an older SDK. Delete {OLD_CONFIG} and {WORKFLOW_DIRECTORY}/{WORKFLOW}, then run bitfab-replay --cloud-init"
|
|
243
237
|
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def setup_workflow(root):
|
|
241
|
+
refuse_old_setup(root)
|
|
244
242
|
name = find_workflow(root)
|
|
245
243
|
if name is None:
|
|
246
244
|
raise ValueError(
|
|
@@ -256,6 +254,12 @@ def workflow_secrets(text):
|
|
|
256
254
|
}
|
|
257
255
|
|
|
258
256
|
|
|
257
|
+
def secrets_page(repo, environment):
|
|
258
|
+
if environment:
|
|
259
|
+
return f"https://github.com/{repo}/settings/environments"
|
|
260
|
+
return f"https://github.com/{repo}/settings/secrets/actions"
|
|
261
|
+
|
|
262
|
+
|
|
259
263
|
ENV_ASSIGNMENT = re.compile(r"[ \t]*(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*=")
|
|
260
264
|
|
|
261
265
|
ENV_ESCAPES = {"n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"}
|
|
@@ -306,10 +310,26 @@ def parse_environment_file(text):
|
|
|
306
310
|
return values
|
|
307
311
|
|
|
308
312
|
|
|
313
|
+
def ensure_environment(repo, environment):
|
|
314
|
+
try:
|
|
315
|
+
api(repo, f"environments/{environment}")
|
|
316
|
+
return "exists"
|
|
317
|
+
except ValueError:
|
|
318
|
+
return "unchecked"
|
|
319
|
+
except RuntimeError as error:
|
|
320
|
+
if getattr(error, "http_status", None) != 404:
|
|
321
|
+
return "unchecked"
|
|
322
|
+
try:
|
|
323
|
+
api(repo, f"environments/{environment}", method="PUT", payload={})
|
|
324
|
+
return "created"
|
|
325
|
+
except RuntimeError:
|
|
326
|
+
return "missing"
|
|
327
|
+
|
|
328
|
+
|
|
309
329
|
def configure_secrets(argv):
|
|
310
330
|
parser = argparse.ArgumentParser(
|
|
311
331
|
prog="bitfab-replay --cloud-secrets",
|
|
312
|
-
description="Copy named values from local environment files into the GitHub Actions secrets the cloud replay workflow reads. Values are piped to
|
|
332
|
+
description="Copy named values from local environment files into the GitHub Actions secrets the cloud replay workflow reads. Values are piped to the GitHub CLI on standard input and never printed, logged, or passed as arguments. Without the GitHub CLI, it lists the secrets to create by hand and where.",
|
|
313
333
|
)
|
|
314
334
|
parser.add_argument(
|
|
315
335
|
"names",
|
|
@@ -324,7 +344,7 @@ def configure_secrets(argv):
|
|
|
324
344
|
)
|
|
325
345
|
parser.add_argument(
|
|
326
346
|
"--environment",
|
|
327
|
-
help="GitHub Environment to set the secrets in; defaults to the workflow's environment",
|
|
347
|
+
help="GitHub Environment to set the secrets in; defaults to the workflow's environment, and is created when it does not exist",
|
|
328
348
|
)
|
|
329
349
|
parser.add_argument(
|
|
330
350
|
"--dry-run",
|
|
@@ -333,14 +353,13 @@ def configure_secrets(argv):
|
|
|
333
353
|
)
|
|
334
354
|
args = parser.parse_args(argv)
|
|
335
355
|
root = root_directory()
|
|
336
|
-
|
|
337
|
-
text = within(root, f"{WORKFLOW_DIRECTORY}/{workflow}").read_text()
|
|
356
|
+
text = within(root, f"{WORKFLOW_DIRECTORY}/{setup_workflow(root)}").read_text()
|
|
338
357
|
targets = workflow_secrets(text)
|
|
339
358
|
names = args.names or list(targets)
|
|
340
359
|
unknown = [name for name in names if name not in targets]
|
|
341
360
|
if unknown:
|
|
342
361
|
raise ValueError(
|
|
343
|
-
f"The workflow does not read {', '.join(unknown)}. Add each one to the
|
|
362
|
+
f"The workflow does not read {', '.join(unknown)}. Add each one to the replay step's env as NAME: ${{{{ secrets.{DEFAULT_SECRET_PREFIX}NAME }}}} first"
|
|
344
363
|
)
|
|
345
364
|
configured = ENVIRONMENT_SETTING.search(text)
|
|
346
365
|
environment = args.environment or (configured[1] if configured else None)
|
|
@@ -355,43 +374,51 @@ def configure_secrets(argv):
|
|
|
355
374
|
for name in names:
|
|
356
375
|
value = values.get(name, values.get(targets[name]))
|
|
357
376
|
if value is None:
|
|
358
|
-
missing.append(name)
|
|
377
|
+
missing.append(targets[name])
|
|
359
378
|
elif not value:
|
|
360
|
-
empty.append(name)
|
|
379
|
+
empty.append(targets[name])
|
|
361
380
|
else:
|
|
362
381
|
found.append(name)
|
|
363
|
-
|
|
364
|
-
where = (
|
|
365
|
-
f"https://github.com/{repo}/settings/environments"
|
|
366
|
-
if environment
|
|
367
|
-
else f"https://github.com/{repo}/settings/secrets/actions"
|
|
368
|
-
)
|
|
369
|
-
raise ValueError(
|
|
370
|
-
f"Copying secrets needs the GitHub CLI, which encrypts each value for GitHub. Install it (https://cli.github.com), or create {', '.join(targets[name] for name in found)} by hand at {where}"
|
|
371
|
-
)
|
|
372
|
-
for name in found:
|
|
373
|
-
if not args.dry_run:
|
|
374
|
-
command(
|
|
375
|
-
[
|
|
376
|
-
"gh",
|
|
377
|
-
"secret",
|
|
378
|
-
"set",
|
|
379
|
-
targets[name],
|
|
380
|
-
"--repo",
|
|
381
|
-
repo,
|
|
382
|
-
*(["--env", environment] if environment else []),
|
|
383
|
-
],
|
|
384
|
-
input=values.get(name, values.get(targets[name])),
|
|
385
|
-
)
|
|
386
|
-
return {
|
|
382
|
+
result = {
|
|
387
383
|
"repository": repo,
|
|
388
384
|
"environment": environment,
|
|
389
385
|
"dryRun": args.dry_run,
|
|
390
|
-
"set": [
|
|
386
|
+
"set": [],
|
|
391
387
|
"missing": missing,
|
|
392
388
|
"empty": empty,
|
|
393
|
-
"next": "Create the missing secrets by hand; empty local values were skipped because an empty secret overrides a working default with nothing; a wrong value only surfaces when the first real replay runs",
|
|
394
389
|
}
|
|
390
|
+
if args.dry_run:
|
|
391
|
+
return {**result, "wouldSet": [targets[name] for name in found]}
|
|
392
|
+
if environment:
|
|
393
|
+
result["environmentStatus"] = ensure_environment(repo, environment)
|
|
394
|
+
if found and shutil.which("gh") is None:
|
|
395
|
+
progress(
|
|
396
|
+
"Setting secrets needs the GitHub CLI (https://cli.github.com), which encrypts each value for GitHub; create the ones listed under createByHand instead"
|
|
397
|
+
)
|
|
398
|
+
return {
|
|
399
|
+
**result,
|
|
400
|
+
"createByHand": [targets[name] for name in found],
|
|
401
|
+
"where": secrets_page(repo, environment),
|
|
402
|
+
}
|
|
403
|
+
token = github_access()["token"]
|
|
404
|
+
for name in found:
|
|
405
|
+
command(
|
|
406
|
+
[
|
|
407
|
+
"gh",
|
|
408
|
+
"secret",
|
|
409
|
+
"set",
|
|
410
|
+
targets[name],
|
|
411
|
+
"--repo",
|
|
412
|
+
repo,
|
|
413
|
+
*(["--env", environment] if environment else []),
|
|
414
|
+
],
|
|
415
|
+
input=values.get(name, values.get(targets[name])),
|
|
416
|
+
env=None if token is None else {**os.environ, "GH_TOKEN": token},
|
|
417
|
+
)
|
|
418
|
+
result["set"] = [targets[name] for name in found]
|
|
419
|
+
if missing or empty:
|
|
420
|
+
result["where"] = secrets_page(repo, environment)
|
|
421
|
+
return result
|
|
395
422
|
|
|
396
423
|
|
|
397
424
|
@functools.cache
|
|
@@ -436,7 +463,7 @@ def stored_git_credential():
|
|
|
436
463
|
return None
|
|
437
464
|
|
|
438
465
|
|
|
439
|
-
def api(repo, suffix, *, method="GET", payload=None):
|
|
466
|
+
def api(repo, suffix, *, method="GET", payload=None, raw=False):
|
|
440
467
|
path = f"repos/{repo}" + (f"/{suffix}" if suffix else "")
|
|
441
468
|
access = github_access()
|
|
442
469
|
data = None if payload is None else json.dumps(payload)
|
|
@@ -449,8 +476,7 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
449
476
|
"github.com",
|
|
450
477
|
"-H",
|
|
451
478
|
f"X-GitHub-Api-Version: {API_VERSION}",
|
|
452
|
-
"--jq",
|
|
453
|
-
"tojson",
|
|
479
|
+
*([] if raw else ["--jq", "tojson"]),
|
|
454
480
|
path,
|
|
455
481
|
"--method",
|
|
456
482
|
method,
|
|
@@ -458,6 +484,8 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
458
484
|
],
|
|
459
485
|
input=data,
|
|
460
486
|
)
|
|
487
|
+
if raw:
|
|
488
|
+
return result
|
|
461
489
|
return json.loads(result) if result else None
|
|
462
490
|
request = urllib.request.Request(
|
|
463
491
|
f"https://api.github.com/{path}",
|
|
@@ -472,8 +500,7 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
472
500
|
},
|
|
473
501
|
)
|
|
474
502
|
try:
|
|
475
|
-
|
|
476
|
-
body = response.read()
|
|
503
|
+
body = fetch(request, follow_without_credentials=raw)
|
|
477
504
|
except urllib.error.HTTPError as error:
|
|
478
505
|
raise CommandError(
|
|
479
506
|
f"GitHub {method} {path.split('?')[0]} failed (HTTP {error.code}) using {access['source']}",
|
|
@@ -488,15 +515,40 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
488
515
|
"set SSL_CERT_FILE to a certificate bundle, or log in with the GitHub CLI (gh auth login), which cloud replay uses instead"
|
|
489
516
|
) from None
|
|
490
517
|
raise CommandError(f"Could not reach api.github.com: {reason}") from None
|
|
518
|
+
if raw:
|
|
519
|
+
return body.decode(errors="replace").strip()
|
|
491
520
|
return json.loads(body) if body.strip() else None
|
|
492
521
|
|
|
493
522
|
|
|
523
|
+
def fetch(request, *, follow_without_credentials):
|
|
524
|
+
if not follow_without_credentials:
|
|
525
|
+
with urllib.request.urlopen(request, timeout=60) as response:
|
|
526
|
+
return response.read()
|
|
527
|
+
try:
|
|
528
|
+
with urllib.request.build_opener(KeepRedirect).open(
|
|
529
|
+
request, timeout=60
|
|
530
|
+
) as response:
|
|
531
|
+
return response.read()
|
|
532
|
+
except urllib.error.HTTPError as error:
|
|
533
|
+
location = error.headers.get("Location")
|
|
534
|
+
if error.code not in (301, 302, 303, 307, 308) or not location:
|
|
535
|
+
raise
|
|
536
|
+
with urllib.request.urlopen(location, timeout=120) as response:
|
|
537
|
+
return response.read()
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def option_name(token):
|
|
541
|
+
return token.split("=", 1)[0]
|
|
542
|
+
|
|
543
|
+
|
|
494
544
|
def split_arguments(argv):
|
|
495
|
-
cloud,
|
|
545
|
+
cloud, switches, replay = {}, set(), []
|
|
496
546
|
index = 0
|
|
497
547
|
while index < len(argv):
|
|
498
548
|
token = argv[index]
|
|
499
|
-
name = token
|
|
549
|
+
name = option_name(token)
|
|
550
|
+
if name in RENAMED_FLAGS:
|
|
551
|
+
raise ValueError(f"{name} is gone; use {RENAMED_FLAGS[name]}")
|
|
500
552
|
if name in CLOUD_VALUE_FLAGS:
|
|
501
553
|
if "=" in token:
|
|
502
554
|
value = token.split("=", 1)[1]
|
|
@@ -506,12 +558,9 @@ def split_arguments(argv):
|
|
|
506
558
|
index += 2
|
|
507
559
|
else:
|
|
508
560
|
raise ValueError(f"{name} needs a value")
|
|
509
|
-
if name
|
|
510
|
-
includes.append(value)
|
|
511
|
-
elif name in cloud:
|
|
561
|
+
if name in cloud:
|
|
512
562
|
raise ValueError(f"{name} was given twice")
|
|
513
|
-
|
|
514
|
-
cloud[name] = value
|
|
563
|
+
cloud[name] = value
|
|
515
564
|
elif token in CLOUD_SWITCHES:
|
|
516
565
|
switches.add(token)
|
|
517
566
|
index += 1
|
|
@@ -520,23 +569,23 @@ def split_arguments(argv):
|
|
|
520
569
|
else:
|
|
521
570
|
replay.append(token)
|
|
522
571
|
index += 1
|
|
523
|
-
return cloud,
|
|
572
|
+
return cloud, switches, replay
|
|
524
573
|
|
|
525
574
|
|
|
526
575
|
def parse(argv):
|
|
527
|
-
cloud,
|
|
528
|
-
|
|
529
|
-
if
|
|
530
|
-
if len(
|
|
531
|
-
raise ValueError("Cloud
|
|
532
|
-
operation =
|
|
533
|
-
execution_id = cloud[
|
|
576
|
+
cloud, switches, replay = split_arguments(argv)
|
|
577
|
+
following = [flag for flag in FOLLOW_FLAGS if flag in cloud]
|
|
578
|
+
if following:
|
|
579
|
+
if len(following) != 1 or len(cloud) != 1 or switches - {"--cloud"} or replay:
|
|
580
|
+
raise ValueError("Cloud follow-up commands take only their execution UUID")
|
|
581
|
+
operation = following[0].removeprefix("--cloud-")
|
|
582
|
+
execution_id = cloud[following[0]]
|
|
534
583
|
else:
|
|
535
584
|
if not replay:
|
|
536
585
|
raise ValueError(
|
|
537
586
|
"Add --cloud to the replay command you run locally, such as bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud"
|
|
538
587
|
)
|
|
539
|
-
if not any(token
|
|
588
|
+
if not any(option_name(token) in SELECTION_FLAGS for token in replay):
|
|
540
589
|
raise ValueError(
|
|
541
590
|
"Select traces with --trace-ids, --dataset-ids, or --resume"
|
|
542
591
|
)
|
|
@@ -553,7 +602,6 @@ def parse(argv):
|
|
|
553
602
|
"operation": operation,
|
|
554
603
|
"id": execution_id,
|
|
555
604
|
"cloud": cloud,
|
|
556
|
-
"includes": includes,
|
|
557
605
|
"switches": switches,
|
|
558
606
|
"replay": replay,
|
|
559
607
|
}
|
|
@@ -573,6 +621,10 @@ def validate_arguments(args):
|
|
|
573
621
|
raise ValueError("Replay options exceed 16000 characters")
|
|
574
622
|
|
|
575
623
|
|
|
624
|
+
def dry_run(args):
|
|
625
|
+
return "--dry-run" in args
|
|
626
|
+
|
|
627
|
+
|
|
576
628
|
def replay_request(root, parsed):
|
|
577
629
|
cwd = Path.cwd().resolve()
|
|
578
630
|
args, files = [], []
|
|
@@ -606,8 +658,6 @@ def replay_request(root, parsed):
|
|
|
606
658
|
}
|
|
607
659
|
if "--cloud-timeout" in parsed["cloud"]:
|
|
608
660
|
request["timeoutMinutes"] = int(parsed["cloud"]["--cloud-timeout"])
|
|
609
|
-
if parsed["switches"] & set(CHECK_SWITCHES):
|
|
610
|
-
request["check"] = True
|
|
611
661
|
return request, files
|
|
612
662
|
|
|
613
663
|
|
|
@@ -634,81 +684,71 @@ def sensitive(path):
|
|
|
634
684
|
)
|
|
635
685
|
|
|
636
686
|
|
|
637
|
-
def snapshot(root, workflow,
|
|
687
|
+
def snapshot(root, workflow, files, request, *, preview=False):
|
|
638
688
|
if git(root, "ls-files", "-u"):
|
|
639
689
|
raise ValueError("Resolve merge conflicts before snapshotting")
|
|
640
690
|
head = git(root, "rev-parse", "HEAD")
|
|
641
|
-
included = []
|
|
642
|
-
for value in includes:
|
|
643
|
-
path = within(root, value)
|
|
644
|
-
if not path.is_file() or path.is_symlink() or (root / value).is_symlink():
|
|
645
|
-
raise ValueError("--cloud-include requires individual regular files")
|
|
646
|
-
ignored = subprocess.run(
|
|
647
|
-
["git", "-C", str(root), "check-ignore", "--quiet", "--", value],
|
|
648
|
-
check=False,
|
|
649
|
-
timeout=15,
|
|
650
|
-
).returncode
|
|
651
|
-
if ignored not in (0, 1):
|
|
652
|
-
raise ValueError("Unable to determine whether the included file is ignored")
|
|
653
|
-
if ignored == 0 or sensitive(value):
|
|
654
|
-
raise ValueError(f"Refusing ignored or credential-like file: {value}")
|
|
655
|
-
included.append(value)
|
|
656
691
|
with tempfile.TemporaryDirectory(prefix="bitfab-cloud-index-") as directory:
|
|
657
692
|
env = {**os.environ, "GIT_INDEX_FILE": str(Path(directory) / "index")}
|
|
658
693
|
git(root, "read-tree", head, env=env)
|
|
659
|
-
git(root, "add", "-
|
|
660
|
-
if included:
|
|
661
|
-
git(root, "--literal-pathspecs", "add", "--", *included, env=env)
|
|
694
|
+
git(root, "add", "-A", "--", ".", env=env)
|
|
662
695
|
entries = git(root, "ls-files", "--stage", "-z", env=env).split("\x00")
|
|
696
|
+
tracked = set()
|
|
663
697
|
for entry in filter(None, entries):
|
|
664
698
|
metadata, path = entry.split("\t", 1)
|
|
699
|
+
tracked.add(path)
|
|
665
700
|
if metadata.startswith("160000"):
|
|
666
701
|
raise ValueError(
|
|
667
|
-
"
|
|
702
|
+
f"{path} is a submodule or nested repository, which cloud snapshots do not support"
|
|
668
703
|
)
|
|
669
704
|
if sensitive(path):
|
|
670
705
|
raise ValueError(
|
|
671
|
-
f"Refusing credential-like
|
|
706
|
+
f"Refusing credential-like file {path}; add it to .gitignore, or git rm --cached it when it is tracked"
|
|
672
707
|
)
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
untracked = [path for path in required if path not in tracked]
|
|
680
|
-
if untracked:
|
|
708
|
+
ignored = [
|
|
709
|
+
path
|
|
710
|
+
for path in [f"{WORKFLOW_DIRECTORY}/{workflow}", *files]
|
|
711
|
+
if path not in tracked
|
|
712
|
+
]
|
|
713
|
+
if ignored:
|
|
681
714
|
raise ValueError(
|
|
682
|
-
"
|
|
683
|
-
+ ", ".join(
|
|
684
|
-
+ "
|
|
715
|
+
"Git ignores "
|
|
716
|
+
+ ", ".join(ignored)
|
|
717
|
+
+ ", so the snapshot cannot include it; stop ignoring it or move it"
|
|
685
718
|
)
|
|
686
719
|
tree = git(root, "write-tree", env=env)
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
)
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
720
|
+
changes = [
|
|
721
|
+
line.split("\t", 1)
|
|
722
|
+
for line in git(
|
|
723
|
+
root, "diff", "--name-status", "--no-renames", head, tree
|
|
724
|
+
).splitlines()
|
|
725
|
+
]
|
|
726
|
+
new_files = sorted(path for status, path in changes if status == "A")
|
|
727
|
+
changed_files = sorted(path for status, path in changes if status != "A")
|
|
728
|
+
sizes = {path: (root / path).lstat().st_size for path in new_files}
|
|
729
|
+
size = sum(sizes.values())
|
|
730
|
+
if size > NEW_FILES_LIMIT:
|
|
731
|
+
largest = sorted(new_files, key=lambda path: -sizes[path])
|
|
732
|
+
raise ValueError(
|
|
733
|
+
f"New files add {size // (1024 * 1024)} MiB to the snapshot, more than {NEW_FILES_LIMIT // (1024 * 1024)} MiB; add large outputs to .gitignore, such as "
|
|
734
|
+
+ ", ".join(largest[:5])
|
|
696
735
|
)
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
736
|
+
summary = {
|
|
737
|
+
"baseSha": head,
|
|
738
|
+
"changedFiles": changed_files,
|
|
739
|
+
"newFiles": new_files,
|
|
740
|
+
}
|
|
741
|
+
if preview:
|
|
742
|
+
return summary
|
|
703
743
|
sha = git(
|
|
704
744
|
root,
|
|
705
745
|
"commit-tree",
|
|
706
746
|
tree,
|
|
707
747
|
"-p",
|
|
708
748
|
head,
|
|
709
|
-
input=f"Bitfab replay {
|
|
749
|
+
input=f"Bitfab replay {request['id']}\n\n{json.dumps(request)}\n",
|
|
710
750
|
)
|
|
711
|
-
return {
|
|
751
|
+
return {**summary, "sha": sha}
|
|
712
752
|
|
|
713
753
|
|
|
714
754
|
def state_directory(root):
|
|
@@ -749,8 +789,7 @@ def find_run(record):
|
|
|
749
789
|
matches = [
|
|
750
790
|
run
|
|
751
791
|
for run in runs["workflow_runs"]
|
|
752
|
-
if run["head_sha"] == record["sha"]
|
|
753
|
-
and run.get("display_title") == f"Bitfab replay {record['id']}"
|
|
792
|
+
if run["head_sha"] == record["sha"] and run["head_branch"] == record["branch"]
|
|
754
793
|
]
|
|
755
794
|
if len(matches) > 1:
|
|
756
795
|
raise ValueError(
|
|
@@ -772,77 +811,115 @@ def status(record, *, fetch_result=True):
|
|
|
772
811
|
return record
|
|
773
812
|
if run["head_sha"] != record["sha"] or run["head_branch"] != record["branch"]:
|
|
774
813
|
raise ValueError("GitHub run does not match the recorded execution")
|
|
775
|
-
if run.get("display_title") != f"Bitfab replay {record['id']}":
|
|
776
|
-
if run["status"] == "completed":
|
|
777
|
-
raise ValueError("GitHub run does not match the recorded execution")
|
|
778
|
-
record.update(runId=run["id"], url=run["html_url"], state=run["status"])
|
|
779
|
-
return record
|
|
780
814
|
record.update(
|
|
781
815
|
runId=run["id"],
|
|
782
816
|
url=run["html_url"],
|
|
783
817
|
state=run["status"],
|
|
784
818
|
conclusion=run.get("conclusion"),
|
|
785
819
|
)
|
|
786
|
-
if
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
820
|
+
if fetch_result and record["state"] == "completed" and "exitCode" not in record:
|
|
821
|
+
outcome(record)
|
|
822
|
+
return record
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def replay_job(record):
|
|
826
|
+
jobs = api(
|
|
827
|
+
record["repository"], f"actions/runs/{record['runId']}/jobs?per_page=100"
|
|
828
|
+
)
|
|
829
|
+
jobs = (jobs or {}).get("jobs", [])
|
|
830
|
+
named = [job for job in jobs if job.get("name") == "replay"]
|
|
831
|
+
return (named or jobs or [None])[0]
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def job_log(record):
|
|
835
|
+
job = replay_job(record)
|
|
836
|
+
if job is None:
|
|
837
|
+
return []
|
|
838
|
+
for attempt in range(6):
|
|
794
839
|
try:
|
|
795
|
-
|
|
840
|
+
text = api(record["repository"], f"actions/jobs/{job['id']}/logs", raw=True)
|
|
841
|
+
return [LOG_TIMESTAMP.sub("", line) for line in text.splitlines()]
|
|
796
842
|
except RuntimeError:
|
|
797
|
-
if
|
|
843
|
+
if attempt == 5:
|
|
798
844
|
raise
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
if success:
|
|
802
|
-
raise ValueError("The GitHub job succeeded but left no replay result")
|
|
803
|
-
record["resultChecked"] = True
|
|
804
|
-
return record
|
|
805
|
-
if (
|
|
806
|
-
result.get("executionId") != record["id"]
|
|
807
|
-
or result.get("commitSha") != record["sha"]
|
|
808
|
-
):
|
|
809
|
-
raise ValueError("Replay result does not match this execution")
|
|
810
|
-
if record["request"].get("check"):
|
|
811
|
-
if result.get("check") != "passed" or not isinstance(
|
|
812
|
-
result.get("resolved"), int
|
|
813
|
-
):
|
|
814
|
-
raise ValueError("Cloud check result does not match this execution")
|
|
815
|
-
record["check"] = "passed"
|
|
816
|
-
record["resolved"] = result["resolved"]
|
|
817
|
-
return record
|
|
818
|
-
if not UUID.fullmatch(result.get("testRunId", "")):
|
|
819
|
-
raise ValueError("Replay result does not match this execution")
|
|
820
|
-
counts = replay_counts(result)
|
|
821
|
-
record["testRunId"] = result["testRunId"]
|
|
822
|
-
if isinstance(result.get("stoppedEarly"), str):
|
|
823
|
-
record["stoppedEarly"] = result["stoppedEarly"]
|
|
824
|
-
if counts is not None:
|
|
825
|
-
record["replayed"], record["errored"] = counts
|
|
826
|
-
return record
|
|
845
|
+
time.sleep(POLL_SECONDS)
|
|
846
|
+
return []
|
|
827
847
|
|
|
828
848
|
|
|
829
|
-
def read_result(
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
849
|
+
def read_result(lines):
|
|
850
|
+
chunks = [
|
|
851
|
+
line[len(RESULT_LINE) :] for line in lines if line.startswith(RESULT_LINE)
|
|
852
|
+
]
|
|
853
|
+
if not chunks:
|
|
854
|
+
return None
|
|
855
|
+
try:
|
|
856
|
+
return json.loads(
|
|
857
|
+
zlib.decompress(base64.b64decode("".join(chunks), validate=True))
|
|
858
|
+
)
|
|
859
|
+
except ValueError as error:
|
|
860
|
+
raise ValueError("The runner left an unreadable replay result") from error
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def replay_output(lines):
|
|
864
|
+
if OUTPUT_BEGIN not in lines:
|
|
865
|
+
return None
|
|
866
|
+
start = lines.index(OUTPUT_BEGIN) + 1
|
|
867
|
+
end = lines.index(OUTPUT_END, start) if OUTPUT_END in lines[start:] else len(lines)
|
|
868
|
+
return lines[start:end]
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def outcome(record):
|
|
872
|
+
lines = job_log(record)
|
|
873
|
+
result = read_result(lines)
|
|
874
|
+
if result is None:
|
|
875
|
+
record["exitCode"] = 1
|
|
876
|
+
return {"record": record, "lines": lines, "result": None}
|
|
877
|
+
if (
|
|
878
|
+
result.get("executionId") != record["id"]
|
|
879
|
+
or result.get("commitSha") != record["sha"]
|
|
880
|
+
):
|
|
881
|
+
raise ValueError("Replay result does not match this execution")
|
|
882
|
+
record["exitCode"] = result["exitCode"]
|
|
883
|
+
parsed = last_json(result.get("stdout", "")) or {}
|
|
884
|
+
test_run = result.get("testRunId") or parsed.get(
|
|
885
|
+
"testRunId", parsed.get("test_run_id")
|
|
886
|
+
)
|
|
887
|
+
if isinstance(test_run, str) and UUID.fullmatch(test_run):
|
|
888
|
+
record["testRunId"] = test_run
|
|
889
|
+
if result.get("stoppedEarly"):
|
|
890
|
+
record["stoppedEarly"] = result["stoppedEarly"]
|
|
891
|
+
return {"record": record, "lines": lines, "result": result}
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def print_outcome(found):
|
|
895
|
+
record, lines, result = found["record"], found["lines"], found["result"]
|
|
896
|
+
if result is None:
|
|
897
|
+
errors = [
|
|
898
|
+
index for index, line in enumerate(lines) if line.startswith("##[error]")
|
|
899
|
+
]
|
|
900
|
+
shown = lines[: errors[0] + 1] if errors else lines
|
|
901
|
+
tail = [line for line in shown if line.strip()][-LOG_TAIL_LINES:]
|
|
902
|
+
for line in tail:
|
|
903
|
+
print(line, file=sys.stderr)
|
|
904
|
+
progress(
|
|
905
|
+
f"The run stopped before the replay finished ({record.get('conclusion')}); the end of its log is above. Full log: {record.get('url')}"
|
|
906
|
+
)
|
|
907
|
+
return 1
|
|
908
|
+
for line in replay_output(lines) or []:
|
|
909
|
+
print(line, file=sys.stderr)
|
|
910
|
+
sys.stderr.flush()
|
|
911
|
+
print(result.get("stdout", ""), end="", flush=True)
|
|
912
|
+
if record.get("stoppedEarly"):
|
|
913
|
+
progress(
|
|
914
|
+
f"Replay stopped early: {record['stoppedEarly']}. Traces that finished are saved"
|
|
915
|
+
+ (
|
|
916
|
+
f" in experiment {record['testRunId']}"
|
|
917
|
+
if record.get("testRunId")
|
|
918
|
+
else ""
|
|
919
|
+
)
|
|
920
|
+
)
|
|
921
|
+
return result["exitCode"] or 1
|
|
922
|
+
return result["exitCode"]
|
|
846
923
|
|
|
847
924
|
|
|
848
925
|
def cleanup(root, record):
|
|
@@ -901,6 +978,123 @@ def preflight(repo, workflow):
|
|
|
901
978
|
)
|
|
902
979
|
|
|
903
980
|
|
|
981
|
+
def describe_snapshot(source):
|
|
982
|
+
changed, new = len(source["changedFiles"]), len(source["newFiles"])
|
|
983
|
+
progress(
|
|
984
|
+
f"Snapshot of {source['baseSha'][:12]} with {changed} changed and {new} new files"
|
|
985
|
+
+ (
|
|
986
|
+
f" ({', '.join(source['newFiles'][:5])}{', ...' if new > 5 else ''})"
|
|
987
|
+
if new
|
|
988
|
+
else ""
|
|
989
|
+
)
|
|
990
|
+
)
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
def submit(root, repo, workflow, parsed, path):
|
|
994
|
+
execution_id = parsed["id"]
|
|
995
|
+
request, files = replay_request(root, parsed)
|
|
996
|
+
if "--cloud-preview" in parsed["switches"]:
|
|
997
|
+
return {
|
|
998
|
+
"preview": True,
|
|
999
|
+
"repository": repo,
|
|
1000
|
+
**snapshot(root, workflow, files, request, preview=True),
|
|
1001
|
+
}
|
|
1002
|
+
if path.exists():
|
|
1003
|
+
record = json.loads(path.read_text())
|
|
1004
|
+
if record["request"] != request or record["repository"].lower() != repo.lower():
|
|
1005
|
+
raise ValueError("Execution ID already belongs to a different request")
|
|
1006
|
+
status(record, fetch_result=False)
|
|
1007
|
+
if record["state"] == "prepared":
|
|
1008
|
+
raise ValueError(
|
|
1009
|
+
"Submission stopped before dispatch. Use --cloud-cleanup, then submit a new execution UUID"
|
|
1010
|
+
)
|
|
1011
|
+
return record
|
|
1012
|
+
preflight(repo, workflow)
|
|
1013
|
+
if dry_run(request["args"]):
|
|
1014
|
+
progress(
|
|
1015
|
+
"--dry-run dispatches a GitHub run that checks secrets and resolves the traces without replaying; it receives the workflow's secrets"
|
|
1016
|
+
)
|
|
1017
|
+
source = snapshot(root, workflow, files, request)
|
|
1018
|
+
describe_snapshot(source)
|
|
1019
|
+
record = {
|
|
1020
|
+
"id": execution_id,
|
|
1021
|
+
"repository": repo,
|
|
1022
|
+
"workflow": workflow,
|
|
1023
|
+
"branch": PREFIX + execution_id,
|
|
1024
|
+
"request": request,
|
|
1025
|
+
"state": "prepared",
|
|
1026
|
+
**source,
|
|
1027
|
+
}
|
|
1028
|
+
save(path, record)
|
|
1029
|
+
progress(f"Execution {execution_id}; resume with --cloud-watch {execution_id}")
|
|
1030
|
+
ref = "refs/heads/" + record["branch"]
|
|
1031
|
+
try:
|
|
1032
|
+
git(
|
|
1033
|
+
root,
|
|
1034
|
+
"push",
|
|
1035
|
+
f"--force-with-lease={ref}:",
|
|
1036
|
+
"origin",
|
|
1037
|
+
f"{record['sha']}:{ref}",
|
|
1038
|
+
)
|
|
1039
|
+
except RuntimeError as error:
|
|
1040
|
+
raise ValueError(
|
|
1041
|
+
f"Pushing the snapshot branch {record['branch']} to origin failed; check that git can push to {repo}"
|
|
1042
|
+
) from error
|
|
1043
|
+
record["state"] = "dispatch_unknown"
|
|
1044
|
+
save(path, record)
|
|
1045
|
+
payload = {"ref": record["branch"]}
|
|
1046
|
+
if DECLARED_INPUTS.search(
|
|
1047
|
+
within(root, f"{WORKFLOW_DIRECTORY}/{workflow}").read_text()
|
|
1048
|
+
):
|
|
1049
|
+
payload["inputs"] = {"execution_id": execution_id, "request": "-"}
|
|
1050
|
+
try:
|
|
1051
|
+
response = api(
|
|
1052
|
+
repo,
|
|
1053
|
+
f"actions/workflows/{workflow}/dispatches",
|
|
1054
|
+
method="POST",
|
|
1055
|
+
payload=payload,
|
|
1056
|
+
)
|
|
1057
|
+
except RuntimeError as error:
|
|
1058
|
+
raise ValueError(
|
|
1059
|
+
f"Dispatching {workflow} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-watch {execution_id}"
|
|
1060
|
+
) from error
|
|
1061
|
+
if isinstance(response, dict) and response.get("workflow_run_id"):
|
|
1062
|
+
record["runId"] = response["workflow_run_id"]
|
|
1063
|
+
record["url"] = response.get("html_url")
|
|
1064
|
+
record["state"] = "queued"
|
|
1065
|
+
return record
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
def report_steps(record, shown):
|
|
1069
|
+
job = replay_job(record)
|
|
1070
|
+
for step in (job or {}).get("steps", []):
|
|
1071
|
+
if step.get("status") != "queued" and step.get("name") not in shown:
|
|
1072
|
+
shown.add(step.get("name"))
|
|
1073
|
+
progress(step.get("name"))
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
def wait(record, path):
|
|
1077
|
+
started = time.monotonic()
|
|
1078
|
+
announced, steps = None, set()
|
|
1079
|
+
while record["state"] != "completed":
|
|
1080
|
+
status(record, fetch_result=False)
|
|
1081
|
+
save(path, record)
|
|
1082
|
+
if record["state"] != announced and record.get("url"):
|
|
1083
|
+
announced = record["state"]
|
|
1084
|
+
progress(f"{announced.replace('_', ' ').capitalize()}: {record['url']}")
|
|
1085
|
+
if record["state"] == "in_progress":
|
|
1086
|
+
report_steps(record, steps)
|
|
1087
|
+
if (
|
|
1088
|
+
record["state"] == "dispatch_unknown"
|
|
1089
|
+
and time.monotonic() - started >= 10 * 60
|
|
1090
|
+
):
|
|
1091
|
+
raise ValueError(
|
|
1092
|
+
f"GitHub has shown no run for this dispatch after 10 minutes. Check Actions, then resume with --cloud-watch {record['id']} or remove the snapshot branch with --cloud-cleanup {record['id']}"
|
|
1093
|
+
)
|
|
1094
|
+
if record["state"] != "completed":
|
|
1095
|
+
time.sleep(POLL_SECONDS)
|
|
1096
|
+
|
|
1097
|
+
|
|
904
1098
|
def run_cli(argv):
|
|
905
1099
|
parsed = parse(argv)
|
|
906
1100
|
operation, execution_id = parsed["operation"], parsed["id"]
|
|
@@ -913,99 +1107,9 @@ def run_cli(argv):
|
|
|
913
1107
|
path = directory / f"{execution_id}.json"
|
|
914
1108
|
with execution_lock(directory, execution_id):
|
|
915
1109
|
if operation == "submit":
|
|
916
|
-
|
|
917
|
-
if "
|
|
918
|
-
return
|
|
919
|
-
"dryRun": True,
|
|
920
|
-
"repository": repo,
|
|
921
|
-
**snapshot(
|
|
922
|
-
root,
|
|
923
|
-
workflow,
|
|
924
|
-
parsed["includes"],
|
|
925
|
-
files,
|
|
926
|
-
execution_id,
|
|
927
|
-
dry_run=True,
|
|
928
|
-
),
|
|
929
|
-
}
|
|
930
|
-
if path.exists():
|
|
931
|
-
record = json.loads(path.read_text())
|
|
932
|
-
if (
|
|
933
|
-
record["request"] != request
|
|
934
|
-
or record["repository"].lower() != repo.lower()
|
|
935
|
-
):
|
|
936
|
-
raise ValueError(
|
|
937
|
-
"Execution ID already belongs to a different request"
|
|
938
|
-
)
|
|
939
|
-
status(record)
|
|
940
|
-
if record["state"] == "prepared":
|
|
941
|
-
raise ValueError(
|
|
942
|
-
"Submission stopped before dispatch. Use --cloud-cleanup, then submit a new execution UUID"
|
|
943
|
-
)
|
|
944
|
-
else:
|
|
945
|
-
preflight(repo, workflow)
|
|
946
|
-
if request.get("check"):
|
|
947
|
-
print(
|
|
948
|
-
"Cloud dry run: dispatching a GitHub run that checks secrets and resolves the traces without replaying. It receives the workflow's secrets. Use --cloud-dry-run to only review the snapshot.",
|
|
949
|
-
file=sys.stderr,
|
|
950
|
-
flush=True,
|
|
951
|
-
)
|
|
952
|
-
source = snapshot(
|
|
953
|
-
root, workflow, parsed["includes"], files, execution_id
|
|
954
|
-
)
|
|
955
|
-
record = {
|
|
956
|
-
"id": execution_id,
|
|
957
|
-
"repository": repo,
|
|
958
|
-
"workflow": workflow,
|
|
959
|
-
"branch": PREFIX + execution_id,
|
|
960
|
-
"request": request,
|
|
961
|
-
"state": "prepared",
|
|
962
|
-
**source,
|
|
963
|
-
}
|
|
964
|
-
if "--fail-on-error" in parsed["switches"]:
|
|
965
|
-
record["failOnError"] = True
|
|
966
|
-
save(path, record)
|
|
967
|
-
print(
|
|
968
|
-
f"Cloud execution {execution_id}. Recover with --cloud-status {execution_id}",
|
|
969
|
-
file=sys.stderr,
|
|
970
|
-
flush=True,
|
|
971
|
-
)
|
|
972
|
-
ref = "refs/heads/" + record["branch"]
|
|
973
|
-
try:
|
|
974
|
-
git(
|
|
975
|
-
root,
|
|
976
|
-
"push",
|
|
977
|
-
f"--force-with-lease={ref}:",
|
|
978
|
-
"origin",
|
|
979
|
-
f"{record['sha']}:{ref}",
|
|
980
|
-
)
|
|
981
|
-
except RuntimeError as error:
|
|
982
|
-
raise ValueError(
|
|
983
|
-
f"Pushing the snapshot branch {record['branch']} to origin failed; check that git can push to {repo}"
|
|
984
|
-
) from error
|
|
985
|
-
record["state"] = "dispatch_unknown"
|
|
986
|
-
save(path, record)
|
|
987
|
-
encoded = base64.b64encode(json.dumps(request).encode()).decode()
|
|
988
|
-
try:
|
|
989
|
-
response = api(
|
|
990
|
-
repo,
|
|
991
|
-
f"actions/workflows/{workflow}/dispatches",
|
|
992
|
-
method="POST",
|
|
993
|
-
payload={
|
|
994
|
-
"ref": record["branch"],
|
|
995
|
-
"inputs": {
|
|
996
|
-
"execution_id": execution_id,
|
|
997
|
-
"request": encoded,
|
|
998
|
-
},
|
|
999
|
-
},
|
|
1000
|
-
)
|
|
1001
|
-
except RuntimeError as error:
|
|
1002
|
-
raise ValueError(
|
|
1003
|
-
f"Dispatching {workflow} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-status {execution_id}"
|
|
1004
|
-
) from error
|
|
1005
|
-
if isinstance(response, dict) and response.get("workflow_run_id"):
|
|
1006
|
-
record["runId"] = response["workflow_run_id"]
|
|
1007
|
-
record["url"] = response.get("html_url")
|
|
1008
|
-
record["state"] = "queued"
|
|
1110
|
+
record = submit(root, repo, workflow, parsed, path)
|
|
1111
|
+
if record.get("preview"):
|
|
1112
|
+
return record, None
|
|
1009
1113
|
else:
|
|
1010
1114
|
if not path.exists():
|
|
1011
1115
|
raise ValueError(
|
|
@@ -1014,7 +1118,7 @@ def run_cli(argv):
|
|
|
1014
1118
|
record = json.loads(path.read_text())
|
|
1015
1119
|
if record["repository"].lower() != repo.lower():
|
|
1016
1120
|
raise ValueError("origin no longer matches the execution repository")
|
|
1017
|
-
status(record, fetch_result=operation
|
|
1121
|
+
status(record, fetch_result=operation == "status")
|
|
1018
1122
|
if operation == "cancel" and record["state"] != "completed":
|
|
1019
1123
|
if not record.get("runId"):
|
|
1020
1124
|
raise ValueError(
|
|
@@ -1023,27 +1127,13 @@ def run_cli(argv):
|
|
|
1023
1127
|
api(repo, f"actions/runs/{record['runId']}/cancel", method="POST")
|
|
1024
1128
|
record["state"] = "cancel_requested"
|
|
1025
1129
|
save(path, record)
|
|
1026
|
-
|
|
1027
|
-
return record
|
|
1028
|
-
if operation == "watch" or (
|
|
1130
|
+
attached = operation == "watch" or (
|
|
1029
1131
|
operation == "submit" and "--cloud-detach" not in parsed["switches"]
|
|
1030
|
-
)
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
if (
|
|
1036
|
-
record["state"] == "dispatch_unknown"
|
|
1037
|
-
and time.monotonic() - started >= 10 * 60
|
|
1038
|
-
):
|
|
1039
|
-
raise ValueError(
|
|
1040
|
-
"GitHub has shown no run for this dispatch after 10 minutes. Check Actions, then resume with --cloud-watch "
|
|
1041
|
-
+ execution_id
|
|
1042
|
-
+ " or remove the snapshot branch with --cloud-cleanup "
|
|
1043
|
-
+ execution_id
|
|
1044
|
-
)
|
|
1045
|
-
if record["state"] != "completed":
|
|
1046
|
-
time.sleep(5)
|
|
1132
|
+
)
|
|
1133
|
+
found = None
|
|
1134
|
+
if attached and record["state"] != "prepared":
|
|
1135
|
+
wait(record, path)
|
|
1136
|
+
found = outcome(record)
|
|
1047
1137
|
if record["state"] == "completed" or (
|
|
1048
1138
|
operation == "cleanup" and record["state"] == "prepared"
|
|
1049
1139
|
):
|
|
@@ -1051,7 +1141,7 @@ def run_cli(argv):
|
|
|
1051
1141
|
elif operation == "cleanup":
|
|
1052
1142
|
raise ValueError("Execution is not completed; no branch was deleted")
|
|
1053
1143
|
save(path, record)
|
|
1054
|
-
return record
|
|
1144
|
+
return record, found
|
|
1055
1145
|
|
|
1056
1146
|
|
|
1057
1147
|
def replay_command():
|
|
@@ -1065,25 +1155,29 @@ def replay_command():
|
|
|
1065
1155
|
or not all(isinstance(part, str) and part for part in value)
|
|
1066
1156
|
):
|
|
1067
1157
|
raise ValueError(
|
|
1068
|
-
"Start the
|
|
1158
|
+
"Start the replay step with the SDK's bitfab-replay command and --cloud-execute, so the runner knows how to start the replay"
|
|
1069
1159
|
)
|
|
1070
1160
|
return value
|
|
1071
1161
|
|
|
1072
1162
|
|
|
1073
|
-
def
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
if
|
|
1163
|
+
def snapshot_request(root, commit):
|
|
1164
|
+
message = git(root, "log", "-1", "--format=%B", commit)
|
|
1165
|
+
_, _, body = message.partition("\n\n")
|
|
1166
|
+
try:
|
|
1167
|
+
request = json.loads(body)
|
|
1168
|
+
except ValueError:
|
|
1169
|
+
request = None
|
|
1170
|
+
if not isinstance(request, dict) or request.get("version") != REQUEST_VERSION:
|
|
1171
|
+
version = request.get("version", 0) if isinstance(request, dict) else 0
|
|
1172
|
+
newer = isinstance(version, int) and version > REQUEST_VERSION
|
|
1081
1173
|
raise ValueError(
|
|
1082
|
-
"This replay was submitted by
|
|
1174
|
+
"This replay was submitted by a "
|
|
1175
|
+
+ ("newer" if newer else "different")
|
|
1176
|
+
+ " SDK than the one the snapshot installs; match the SDK you run locally to the one in the lockfile"
|
|
1083
1177
|
)
|
|
1084
|
-
if
|
|
1178
|
+
if os.environ.get("GITHUB_REF_NAME") != PREFIX + str(request.get("id")):
|
|
1085
1179
|
raise ValueError(
|
|
1086
|
-
"
|
|
1180
|
+
"The runner replays only the snapshot branch its request names; start replays with bitfab-replay --cloud"
|
|
1087
1181
|
)
|
|
1088
1182
|
validate_arguments(request["args"])
|
|
1089
1183
|
return request
|
|
@@ -1092,45 +1186,43 @@ def dispatched_request():
|
|
|
1092
1186
|
def execute():
|
|
1093
1187
|
if os.environ.get("GITHUB_RUN_ATTEMPT") != "1":
|
|
1094
1188
|
raise ValueError("Submit a new replay instead of rerunning an Actions job")
|
|
1095
|
-
request = dispatched_request()
|
|
1096
1189
|
root = root_directory()
|
|
1097
1190
|
commit = os.environ["GITHUB_SHA"]
|
|
1098
1191
|
if git(root, "rev-parse", "HEAD") != commit:
|
|
1099
1192
|
raise ValueError("Runner checkout does not match the dispatched snapshot SHA")
|
|
1193
|
+
request = snapshot_request(root, commit)
|
|
1100
1194
|
directory = within(root, request["cwd"])
|
|
1101
1195
|
check_secrets(root)
|
|
1102
|
-
names = {value
|
|
1196
|
+
names = {option_name(value) for value in request["args"]}
|
|
1103
1197
|
args = [
|
|
1104
1198
|
*replay_command(),
|
|
1105
1199
|
*request["args"],
|
|
1106
1200
|
*([] if names & set(CODE_CHANGE_FLAGS) else ["--no-code-change"]),
|
|
1107
1201
|
]
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
summary = {**identity, **run_check(directory, args)}
|
|
1111
|
-
write_result(summary)
|
|
1112
|
-
return summary
|
|
1202
|
+
if dry_run(request["args"]):
|
|
1203
|
+
run_check_command(directory)
|
|
1113
1204
|
experiment = {}
|
|
1114
1205
|
previous = signal.signal(signal.SIGTERM, raise_interrupt)
|
|
1206
|
+
print(OUTPUT_BEGIN, flush=True)
|
|
1115
1207
|
try:
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
}
|
|
1120
|
-
except BaseException as error:
|
|
1121
|
-
if UUID.fullmatch(experiment.get("id", "")):
|
|
1122
|
-
write_result(
|
|
1123
|
-
{
|
|
1124
|
-
**identity,
|
|
1125
|
-
"testRunId": experiment["id"],
|
|
1126
|
-
"stoppedEarly": (str(error) or type(error).__name__)[:300],
|
|
1127
|
-
}
|
|
1128
|
-
)
|
|
1129
|
-
raise
|
|
1208
|
+
code, stdout, stopped = run_command(
|
|
1209
|
+
directory, args, request.get("timeoutMinutes"), experiment
|
|
1210
|
+
)
|
|
1130
1211
|
finally:
|
|
1212
|
+
print(OUTPUT_END, flush=True)
|
|
1131
1213
|
signal.signal(signal.SIGTERM, previous)
|
|
1132
|
-
|
|
1133
|
-
|
|
1214
|
+
result = {
|
|
1215
|
+
"executionId": request["id"],
|
|
1216
|
+
"commitSha": commit,
|
|
1217
|
+
"exitCode": code,
|
|
1218
|
+
"stdout": stdout,
|
|
1219
|
+
}
|
|
1220
|
+
if stopped:
|
|
1221
|
+
result["stoppedEarly"] = stopped
|
|
1222
|
+
if UUID.fullmatch(experiment.get("id", "")):
|
|
1223
|
+
result["testRunId"] = experiment["id"]
|
|
1224
|
+
write_result(result)
|
|
1225
|
+
return result
|
|
1134
1226
|
|
|
1135
1227
|
|
|
1136
1228
|
def running_workflow(root):
|
|
@@ -1153,26 +1245,16 @@ def check_secrets(root):
|
|
|
1153
1245
|
)
|
|
1154
1246
|
|
|
1155
1247
|
|
|
1156
|
-
def
|
|
1248
|
+
def run_check_command(directory):
|
|
1157
1249
|
check = os.environ.get(CHECK_COMMAND_ENV, "").strip()
|
|
1158
|
-
if check:
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
).
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
items = result.get("items")
|
|
1167
|
-
if not isinstance(items, list):
|
|
1168
|
-
raise ValueError("The replay dry run did not return its resolved items")
|
|
1169
|
-
errors = item_errors(items)
|
|
1170
|
-
report_item_errors(errors)
|
|
1171
|
-
if errors:
|
|
1172
|
-
raise ValueError(
|
|
1173
|
-
f"{len(errors)} of {len(items)} traces failed to resolve; the errors are above"
|
|
1174
|
-
)
|
|
1175
|
-
return {"check": "passed", "resolved": len(items)}
|
|
1250
|
+
if not check:
|
|
1251
|
+
return
|
|
1252
|
+
print(f"Running {CHECK_COMMAND_ENV}: {check}", flush=True)
|
|
1253
|
+
code = subprocess.run(
|
|
1254
|
+
shlex.split(check), cwd=directory, stdin=subprocess.DEVNULL, check=False
|
|
1255
|
+
).returncode
|
|
1256
|
+
if code:
|
|
1257
|
+
raise ValueError(f"{CHECK_COMMAND_ENV} exited {code}; its output is above")
|
|
1176
1258
|
|
|
1177
1259
|
|
|
1178
1260
|
def item_errors(items):
|
|
@@ -1200,24 +1282,16 @@ def item_errors(items):
|
|
|
1200
1282
|
return errors
|
|
1201
1283
|
|
|
1202
1284
|
|
|
1203
|
-
def
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
sys.stderr.flush()
|
|
1214
|
-
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
1215
|
-
if summary:
|
|
1216
|
-
with Path(summary).open("a") as file:
|
|
1217
|
-
file.write(
|
|
1218
|
-
"\n#### Errored items\n\n"
|
|
1219
|
-
+ "".join(f"- `{line.replace('`', chr(39))}`\n" for line in lines)
|
|
1220
|
-
)
|
|
1285
|
+
def item_errored(item):
|
|
1286
|
+
return isinstance(item, dict) and any(
|
|
1287
|
+
item.get(field) is not None for field in ITEM_ERROR_FIELDS
|
|
1288
|
+
)
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
def carried_over(item):
|
|
1292
|
+
return isinstance(item, dict) and (
|
|
1293
|
+
item.get("carriedOver") is True or item.get("carried_over") is True
|
|
1294
|
+
)
|
|
1221
1295
|
|
|
1222
1296
|
|
|
1223
1297
|
def raise_interrupt(signum, frame):
|
|
@@ -1226,67 +1300,69 @@ def raise_interrupt(signum, frame):
|
|
|
1226
1300
|
|
|
1227
1301
|
def forward_stderr(stream, experiment):
|
|
1228
1302
|
for line in iter(stream.readline, b""):
|
|
1229
|
-
sys.
|
|
1230
|
-
sys.
|
|
1303
|
+
sys.stdout.write(line.decode(errors="replace"))
|
|
1304
|
+
sys.stdout.flush()
|
|
1231
1305
|
if "id" not in experiment:
|
|
1232
1306
|
match = EXPERIMENT_LINE.match(line)
|
|
1233
1307
|
if match:
|
|
1234
1308
|
experiment["id"] = match[1].decode()
|
|
1235
1309
|
|
|
1236
1310
|
|
|
1237
|
-
def
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
if "
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
f"Commit: `{summary['commitSha']}`",
|
|
1250
|
-
f"Stopped early: {summary['stoppedEarly']}. Traces that finished are saved in this test run."
|
|
1251
|
-
if "stoppedEarly" in summary
|
|
1252
|
-
else f"Replayed: {summary['replayed']}, errored: {summary['errored']}",
|
|
1253
|
-
]
|
|
1254
|
-
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as file:
|
|
1255
|
-
file.write(f"### {heading}\n\n" + "\n\n".join(lines) + "\n")
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
def run_replay(directory, args, timeout, experiment):
|
|
1259
|
-
result = run_command(directory, args, timeout, experiment)
|
|
1260
|
-
test_run = result.get("testRunId", result.get("test_run_id"))
|
|
1261
|
-
if not isinstance(test_run, str) or not UUID.fullmatch(test_run):
|
|
1262
|
-
raise ValueError("Replay did not return a valid persisted test run UUID")
|
|
1263
|
-
items = result.get("items")
|
|
1264
|
-
if not isinstance(items, list):
|
|
1265
|
-
raise ValueError("Replay did not return its replayed items")
|
|
1266
|
-
replayed = [item for item in items if not carried_over(item)]
|
|
1267
|
-
errors = item_errors(replayed)
|
|
1268
|
-
report_item_errors(errors)
|
|
1269
|
-
return {"testRunId": test_run, "replayed": len(replayed), "errored": len(errors)}
|
|
1311
|
+
def last_json(text):
|
|
1312
|
+
decoder = json.JSONDecoder()
|
|
1313
|
+
result = None
|
|
1314
|
+
for offset in [0, *[i + 1 for i, value in enumerate(text) if value == "\n"]]:
|
|
1315
|
+
if text.startswith("{", offset):
|
|
1316
|
+
try:
|
|
1317
|
+
value, end = decoder.raw_decode(text, offset)
|
|
1318
|
+
if not text[end:].strip() and isinstance(value, dict):
|
|
1319
|
+
result = value
|
|
1320
|
+
except json.JSONDecodeError:
|
|
1321
|
+
pass
|
|
1322
|
+
return result
|
|
1270
1323
|
|
|
1271
1324
|
|
|
1272
|
-
def
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1325
|
+
def write_result(result):
|
|
1326
|
+
encoded = base64.b64encode(zlib.compress(json.dumps(result).encode())).decode()
|
|
1327
|
+
for start in range(0, len(encoded), RESULT_CHUNK):
|
|
1328
|
+
print(RESULT_LINE + encoded[start : start + RESULT_CHUNK])
|
|
1329
|
+
sys.stdout.flush()
|
|
1330
|
+
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
1331
|
+
if summary:
|
|
1332
|
+
with Path(summary).open("a") as file:
|
|
1333
|
+
file.write(summary_markdown(result))
|
|
1276
1334
|
|
|
1277
1335
|
|
|
1278
|
-
def
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1336
|
+
def summary_markdown(result):
|
|
1337
|
+
parsed = last_json(result["stdout"]) or {}
|
|
1338
|
+
test_run = result.get("testRunId") or parsed.get(
|
|
1339
|
+
"testRunId", parsed.get("test_run_id")
|
|
1340
|
+
)
|
|
1341
|
+
items = parsed.get("items") if isinstance(parsed.get("items"), list) else []
|
|
1342
|
+
replayed = [item for item in items if not carried_over(item)]
|
|
1343
|
+
errors = item_errors(replayed)
|
|
1344
|
+
lines = [f"Commit: `{result['commitSha']}`", f"Exit code: {result['exitCode']}"]
|
|
1345
|
+
if test_run:
|
|
1346
|
+
lines.insert(0, f"Test run: `{test_run}`")
|
|
1347
|
+
if result.get("stoppedEarly"):
|
|
1348
|
+
lines.append(f"Stopped early: {result['stoppedEarly']}")
|
|
1349
|
+
elif items:
|
|
1350
|
+
lines.append(f"Replayed: {len(replayed)}, errored: {len(errors)}")
|
|
1351
|
+
text = "### Bitfab replay\n\n" + "\n\n".join(lines) + "\n"
|
|
1352
|
+
if errors:
|
|
1353
|
+
shown = [
|
|
1354
|
+
f"trace {trace}: {error}" for trace, error in errors[:ITEM_ERROR_LINES]
|
|
1355
|
+
]
|
|
1356
|
+
if len(errors) > len(shown):
|
|
1357
|
+
shown.append(f"and {len(errors) - len(shown)} more errored items")
|
|
1358
|
+
text += "\n#### Errored items\n\n" + "".join(
|
|
1359
|
+
f"- `{line.replace('`', chr(39))}`\n" for line in shown
|
|
1286
1360
|
)
|
|
1361
|
+
return text
|
|
1287
1362
|
|
|
1288
1363
|
|
|
1289
1364
|
def run_command(directory, args, timeout, experiment):
|
|
1365
|
+
stopped = None
|
|
1290
1366
|
with tempfile.TemporaryFile() as output:
|
|
1291
1367
|
with subprocess.Popen(
|
|
1292
1368
|
args,
|
|
@@ -1303,84 +1379,37 @@ def run_command(directory, args, timeout, experiment):
|
|
|
1303
1379
|
try:
|
|
1304
1380
|
deadline = None if timeout is None else time.monotonic() + timeout * 60
|
|
1305
1381
|
while child.poll() is None:
|
|
1306
|
-
if os.fstat(output.fileno()).st_size >
|
|
1382
|
+
if os.fstat(output.fileno()).st_size > OUTPUT_LIMIT:
|
|
1307
1383
|
raise ValueError("Replay output exceeded 16 MiB")
|
|
1308
1384
|
if deadline is not None and time.monotonic() >= deadline:
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1385
|
+
stopped = f"it reached the {timeout}-minute --cloud-timeout"
|
|
1386
|
+
stop(child)
|
|
1387
|
+
break
|
|
1312
1388
|
time.sleep(0.1)
|
|
1313
|
-
|
|
1389
|
+
except KeyboardInterrupt:
|
|
1390
|
+
stopped = "the GitHub run was cancelled"
|
|
1391
|
+
stop(child)
|
|
1314
1392
|
except BaseException:
|
|
1315
|
-
|
|
1316
|
-
os.killpg(child.pid, signal.SIGTERM)
|
|
1317
|
-
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1318
|
-
child.wait(timeout=30)
|
|
1319
|
-
with contextlib.suppress(ProcessLookupError):
|
|
1320
|
-
os.killpg(child.pid, signal.SIGKILL)
|
|
1321
|
-
child.wait()
|
|
1393
|
+
stop(child)
|
|
1322
1394
|
raise
|
|
1323
1395
|
finally:
|
|
1324
1396
|
reader.join(timeout=5)
|
|
1325
|
-
|
|
1397
|
+
code = child.returncode
|
|
1398
|
+
if output.tell() > OUTPUT_LIMIT:
|
|
1326
1399
|
raise ValueError("Replay output exceeded 16 MiB")
|
|
1327
1400
|
output.seek(0)
|
|
1328
1401
|
text = output.read().decode(errors="replace")
|
|
1329
|
-
if code
|
|
1330
|
-
print_output_tail(text)
|
|
1331
|
-
raise ValueError(f"Replay command exited {code}; its output is above")
|
|
1332
|
-
decoder = json.JSONDecoder()
|
|
1333
|
-
result = None
|
|
1334
|
-
for offset in [0, *[i + 1 for i, value in enumerate(text) if value == "\n"]]:
|
|
1335
|
-
if text.startswith("{", offset):
|
|
1336
|
-
try:
|
|
1337
|
-
value, end = decoder.raw_decode(text, offset)
|
|
1338
|
-
if not text[end:].strip() and isinstance(value, dict):
|
|
1339
|
-
result = value
|
|
1340
|
-
except json.JSONDecodeError:
|
|
1341
|
-
pass
|
|
1342
|
-
if result is None:
|
|
1343
|
-
print_output_tail(text)
|
|
1344
|
-
raise ValueError("Replay did not print a JSON result; its output is above")
|
|
1345
|
-
return result
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
def item_errored(item):
|
|
1349
|
-
return isinstance(item, dict) and any(
|
|
1350
|
-
item.get(field) is not None for field in ITEM_ERROR_FIELDS
|
|
1351
|
-
)
|
|
1402
|
+
return (code if code >= 0 else 1), text, stopped
|
|
1352
1403
|
|
|
1353
1404
|
|
|
1354
|
-
def
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
)
|
|
1363
|
-
or errored > replayed
|
|
1364
|
-
):
|
|
1365
|
-
raise ValueError("Replay result has invalid item counts")
|
|
1366
|
-
return replayed, errored
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
def report_errored_items(record):
|
|
1370
|
-
replayed, errored = record.get("replayed"), record.get("errored")
|
|
1371
|
-
if not errored:
|
|
1372
|
-
return 0
|
|
1373
|
-
if errored == replayed:
|
|
1374
|
-
print(
|
|
1375
|
-
f"Cloud replay: every replayed trace errored ({errored} of {replayed}), so nothing ran successfully. Open test run {record.get('testRunId')} to see why.",
|
|
1376
|
-
file=sys.stderr,
|
|
1377
|
-
)
|
|
1378
|
-
return 1
|
|
1379
|
-
print(
|
|
1380
|
-
f"Cloud replay: {errored} of {replayed} replayed traces errored. Open test run {record.get('testRunId')} to see why.",
|
|
1381
|
-
file=sys.stderr,
|
|
1382
|
-
)
|
|
1383
|
-
return 0
|
|
1405
|
+
def stop(child):
|
|
1406
|
+
with contextlib.suppress(ProcessLookupError):
|
|
1407
|
+
os.killpg(child.pid, signal.SIGTERM)
|
|
1408
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1409
|
+
child.wait(timeout=30)
|
|
1410
|
+
with contextlib.suppress(ProcessLookupError):
|
|
1411
|
+
os.killpg(child.pid, signal.SIGKILL)
|
|
1412
|
+
child.wait()
|
|
1384
1413
|
|
|
1385
1414
|
|
|
1386
1415
|
PLAIN_SCALAR = re.compile(r"[A-Za-z0-9_./][A-Za-z0-9_ ./@*+=,()-]*")
|
|
@@ -1491,30 +1520,26 @@ def typescript_project(root, start):
|
|
|
1491
1520
|
"yarn": "yarn install --frozen-lockfile",
|
|
1492
1521
|
"npm": "npm ci" if lockfile else "npm install",
|
|
1493
1522
|
}[manager]
|
|
1494
|
-
node = {
|
|
1495
|
-
"name": "Set up Node.js",
|
|
1496
|
-
"uses": ACTIONS["node"],
|
|
1497
|
-
"with": {
|
|
1498
|
-
**version_input(root, start, "node", [".nvmrc", ".node-version"], "lts/*"),
|
|
1499
|
-
**(
|
|
1500
|
-
{"cache": manager}
|
|
1501
|
-
if manager in ("pnpm", "yarn", "npm") and lockfile
|
|
1502
|
-
else {}
|
|
1503
|
-
),
|
|
1504
|
-
},
|
|
1505
|
-
}
|
|
1506
1523
|
steps = []
|
|
1507
1524
|
if manager == "pnpm":
|
|
1508
|
-
steps.append({"
|
|
1525
|
+
steps.append({"uses": ACTIONS["pnpm"]})
|
|
1509
1526
|
if manager == "bun":
|
|
1510
|
-
steps.append({"
|
|
1527
|
+
steps.append({"uses": ACTIONS["bun"]})
|
|
1511
1528
|
steps += [
|
|
1512
|
-
node,
|
|
1513
1529
|
{
|
|
1514
|
-
"
|
|
1515
|
-
|
|
1516
|
-
|
|
1530
|
+
"uses": ACTIONS["node"],
|
|
1531
|
+
"with": {
|
|
1532
|
+
**version_input(
|
|
1533
|
+
root, start, "node", [".nvmrc", ".node-version"], "lts/*"
|
|
1534
|
+
),
|
|
1535
|
+
**(
|
|
1536
|
+
{"cache": manager}
|
|
1537
|
+
if manager in ("pnpm", "yarn", "npm") and lockfile
|
|
1538
|
+
else {}
|
|
1539
|
+
),
|
|
1540
|
+
},
|
|
1517
1541
|
},
|
|
1542
|
+
{**step_directory(root, directory or start), "run": install},
|
|
1518
1543
|
]
|
|
1519
1544
|
run = {
|
|
1520
1545
|
"pnpm": "pnpm exec bitfab-replay",
|
|
@@ -1531,20 +1556,18 @@ def python_project(root, start):
|
|
|
1531
1556
|
)
|
|
1532
1557
|
location = step_directory(root, directory or start)
|
|
1533
1558
|
python = {
|
|
1534
|
-
"name": "Set up Python",
|
|
1535
1559
|
"uses": ACTIONS["python"],
|
|
1536
1560
|
"with": version_input(root, start, "python", [".python-version"], "3.12"),
|
|
1537
1561
|
}
|
|
1538
1562
|
if lockfile == "uv.lock":
|
|
1539
1563
|
return [
|
|
1540
|
-
{"
|
|
1541
|
-
{
|
|
1564
|
+
{"uses": ACTIONS["uv"]},
|
|
1565
|
+
{**location, "run": "uv sync --frozen"},
|
|
1542
1566
|
], "uv run bitfab-replay"
|
|
1543
1567
|
if lockfile == "poetry.lock":
|
|
1544
1568
|
return [
|
|
1545
1569
|
python,
|
|
1546
1570
|
{
|
|
1547
|
-
"name": "Install dependencies",
|
|
1548
1571
|
**location,
|
|
1549
1572
|
"run": "pipx install poetry && poetry install --no-interaction",
|
|
1550
1573
|
},
|
|
@@ -1554,21 +1577,14 @@ def python_project(root, start):
|
|
|
1554
1577
|
if lockfile == "requirements.txt"
|
|
1555
1578
|
else "pip install ."
|
|
1556
1579
|
)
|
|
1557
|
-
return [
|
|
1558
|
-
python,
|
|
1559
|
-
{"name": "Install dependencies", **location, "run": install},
|
|
1560
|
-
], "bitfab-replay"
|
|
1580
|
+
return [python, {**location, "run": install}], "bitfab-replay"
|
|
1561
1581
|
|
|
1562
1582
|
|
|
1563
1583
|
def ruby_project(root, start):
|
|
1564
1584
|
directory, _ = find_upward(root, start, ["Gemfile"])
|
|
1565
1585
|
location = step_directory(root, directory or start)
|
|
1566
1586
|
return [
|
|
1567
|
-
{
|
|
1568
|
-
"name": "Set up Ruby",
|
|
1569
|
-
"uses": ACTIONS["ruby"],
|
|
1570
|
-
"with": {"bundler-cache": True, **location},
|
|
1571
|
-
}
|
|
1587
|
+
{"uses": ACTIONS["ruby"], "with": {"bundler-cache": True, **location}}
|
|
1572
1588
|
], "bundle exec bitfab-replay"
|
|
1573
1589
|
|
|
1574
1590
|
|
|
@@ -1579,7 +1595,30 @@ def go_project(root, start):
|
|
|
1579
1595
|
if name
|
|
1580
1596
|
else {"go-version": "stable"}
|
|
1581
1597
|
)
|
|
1582
|
-
return [{"
|
|
1598
|
+
return [{"uses": ACTIONS["go"], "with": version}], go_registry_command(root, start)
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
def go_registry_command(root, start):
|
|
1602
|
+
programs = set()
|
|
1603
|
+
for directory, folders, names in os.walk(root):
|
|
1604
|
+
folders[:] = [
|
|
1605
|
+
folder
|
|
1606
|
+
for folder in folders
|
|
1607
|
+
if not folder.startswith(".")
|
|
1608
|
+
and folder not in ("node_modules", "vendor", "testdata")
|
|
1609
|
+
]
|
|
1610
|
+
for name in names:
|
|
1611
|
+
if not name.endswith(".go") or name.endswith("_test.go"):
|
|
1612
|
+
continue
|
|
1613
|
+
text = (Path(directory) / name).read_text(errors="replace")
|
|
1614
|
+
if "RunCloudReplayCLI(" in text and re.search(
|
|
1615
|
+
r"^package main\b", text, re.MULTILINE
|
|
1616
|
+
):
|
|
1617
|
+
programs.add(Path(directory))
|
|
1618
|
+
if len(programs) != 1:
|
|
1619
|
+
return None
|
|
1620
|
+
relative = os.path.relpath(programs.pop(), start)
|
|
1621
|
+
return f"go run {relative if relative.startswith('..') else './' + relative}"
|
|
1583
1622
|
|
|
1584
1623
|
|
|
1585
1624
|
PROJECTS = {
|
|
@@ -1608,47 +1647,22 @@ def push_triggers(root, workflow):
|
|
|
1608
1647
|
return found
|
|
1609
1648
|
|
|
1610
1649
|
|
|
1611
|
-
def
|
|
1612
|
-
return "${{ " + kind + "." + name + " }}"
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
def secret_env(names, prefix):
|
|
1616
|
-
return {name: reference("secrets", prefix + name) for name in names}
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
def replay_step(run, directory, env):
|
|
1620
|
-
return {
|
|
1621
|
-
"name": "Replay",
|
|
1622
|
-
**({} if directory == "." else {"working-directory": directory}),
|
|
1623
|
-
"run": f"{run} --cloud-execute",
|
|
1624
|
-
"env": env,
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
def workflow_document(steps, replay, *, runs_on="ubuntu-24.04", environment=None):
|
|
1650
|
+
def workflow_document(steps, run, directory, env, *, runs_on, environment):
|
|
1629
1651
|
job = {"runs-on": runs_on}
|
|
1630
1652
|
if environment:
|
|
1631
1653
|
job["environment"] = environment
|
|
1632
1654
|
job["steps"] = [
|
|
1655
|
+
{"uses": ACTIONS["checkout"], "with": {"persist-credentials": False}},
|
|
1656
|
+
*steps,
|
|
1633
1657
|
{
|
|
1634
|
-
|
|
1635
|
-
"
|
|
1636
|
-
"
|
|
1658
|
+
**({} if directory == "." else {"working-directory": directory}),
|
|
1659
|
+
"run": f"{run} --cloud-execute",
|
|
1660
|
+
"env": env,
|
|
1637
1661
|
},
|
|
1638
|
-
*steps,
|
|
1639
|
-
replay,
|
|
1640
1662
|
]
|
|
1641
1663
|
return {
|
|
1642
|
-
"name": "Bitfab
|
|
1643
|
-
"
|
|
1644
|
-
"on": {
|
|
1645
|
-
"workflow_dispatch": {
|
|
1646
|
-
"inputs": {
|
|
1647
|
-
"execution_id": {"required": True, "type": "string"},
|
|
1648
|
-
"request": {"required": True, "type": "string"},
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
},
|
|
1664
|
+
"name": "Bitfab replay",
|
|
1665
|
+
"on": "workflow_dispatch",
|
|
1652
1666
|
"permissions": {"contents": "read"},
|
|
1653
1667
|
"jobs": {"replay": job},
|
|
1654
1668
|
}
|
|
@@ -1657,7 +1671,7 @@ def workflow_document(steps, replay, *, runs_on="ubuntu-24.04", environment=None
|
|
|
1657
1671
|
def initialize(argv):
|
|
1658
1672
|
parser = argparse.ArgumentParser(
|
|
1659
1673
|
prog="bitfab-replay --cloud-init",
|
|
1660
|
-
description="Write .github/workflows/bitfab-replay.yml, the only file cloud replay keeps in the repository. It detects the runtime, package manager, and
|
|
1674
|
+
description="Write .github/workflows/bitfab-replay.yml, the only file cloud replay keeps in the repository, and list what is left to do. It detects the runtime, package manager, install command, and replay command from the directory you run it in. An existing workflow is left alone; edit it directly to change it.",
|
|
1661
1675
|
)
|
|
1662
1676
|
parser.add_argument(
|
|
1663
1677
|
"--secret",
|
|
@@ -1677,24 +1691,24 @@ def initialize(argv):
|
|
|
1677
1691
|
)
|
|
1678
1692
|
parser.add_argument(
|
|
1679
1693
|
"--run",
|
|
1680
|
-
help='Command that starts the SDK\'s bitfab-replay on the runner, such as "pnpm exec bitfab-replay"; detected for
|
|
1694
|
+
help='Command that starts the SDK\'s bitfab-replay on the runner, such as "pnpm exec bitfab-replay"; detected for every SDK, including the Go program that calls RunCloudReplayCLI',
|
|
1681
1695
|
)
|
|
1682
1696
|
parser.add_argument("--runs-on", default="ubuntu-24.04", help="Runner label")
|
|
1683
1697
|
parser.add_argument(
|
|
1684
1698
|
"--check",
|
|
1685
|
-
help='Command the runner also runs
|
|
1699
|
+
help='Command the runner also runs on a --cloud --dry-run, such as "node scripts/checkBucket.js"',
|
|
1686
1700
|
)
|
|
1687
1701
|
args = parser.parse_args(argv)
|
|
1688
1702
|
root = root_directory()
|
|
1689
|
-
|
|
1690
|
-
return migrate(root, args)
|
|
1703
|
+
refuse_old_setup(root)
|
|
1691
1704
|
existing = find_workflow(root)
|
|
1692
1705
|
if existing is not None:
|
|
1693
1706
|
return {
|
|
1694
1707
|
"files": [],
|
|
1695
|
-
"updated": False,
|
|
1696
1708
|
"workflow": f"{WORKFLOW_DIRECTORY}/{existing}",
|
|
1697
|
-
"next":
|
|
1709
|
+
"next": [
|
|
1710
|
+
"Already set up. Edit the workflow directly to change its install steps, secrets, runner, or Environment"
|
|
1711
|
+
],
|
|
1698
1712
|
}
|
|
1699
1713
|
prefix = args.secret_prefix
|
|
1700
1714
|
if prefix and not re.fullmatch(r"[A-Z][A-Z0-9_]*_", prefix):
|
|
@@ -1713,191 +1727,83 @@ def initialize(argv):
|
|
|
1713
1727
|
raise ValueError(
|
|
1714
1728
|
'Pass --run with the command that starts your registry program on the runner, such as --run "go run ./cmd/registry"'
|
|
1715
1729
|
)
|
|
1716
|
-
env =
|
|
1730
|
+
env = {name: "${{ secrets." + prefix + name + " }}" for name in names}
|
|
1717
1731
|
if args.check:
|
|
1718
1732
|
env[CHECK_COMMAND_ENV] = args.check
|
|
1719
|
-
workflow = workflow_document(
|
|
1720
|
-
steps,
|
|
1721
|
-
replay_step(run, directory, env),
|
|
1722
|
-
runs_on=args.runs_on,
|
|
1723
|
-
environment=args.environment,
|
|
1724
|
-
)
|
|
1725
1733
|
path = f"{WORKFLOW_DIRECTORY}/{WORKFLOW}"
|
|
1726
1734
|
target = within(root, path)
|
|
1727
1735
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1728
1736
|
with target.open("x") as file:
|
|
1729
|
-
file.write(
|
|
1737
|
+
file.write(
|
|
1738
|
+
to_yaml(
|
|
1739
|
+
workflow_document(
|
|
1740
|
+
steps,
|
|
1741
|
+
run,
|
|
1742
|
+
directory,
|
|
1743
|
+
env,
|
|
1744
|
+
runs_on=args.runs_on,
|
|
1745
|
+
environment=args.environment,
|
|
1746
|
+
)
|
|
1747
|
+
)
|
|
1748
|
+
)
|
|
1749
|
+
secrets = sorted(prefix + name for name in names)
|
|
1750
|
+
triggers = push_triggers(root, WORKFLOW)
|
|
1751
|
+
try:
|
|
1752
|
+
where = secrets_page(repository(root), args.environment)
|
|
1753
|
+
except (ValueError, RuntimeError):
|
|
1754
|
+
where = "the repository's Settings, Secrets and variables, Actions"
|
|
1730
1755
|
return {
|
|
1731
1756
|
"files": [path],
|
|
1732
1757
|
"detected": {
|
|
1733
1758
|
"language": language,
|
|
1734
1759
|
"install": [step.get("run") or step["uses"] for step in steps],
|
|
1735
|
-
"replay":
|
|
1760
|
+
"replay": f"{run} --cloud-execute",
|
|
1736
1761
|
},
|
|
1737
|
-
"secrets":
|
|
1738
|
-
"
|
|
1739
|
-
"
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
removed = []
|
|
1753
|
-
for old in (OLD_CONFIG, OLD_SCRIPT):
|
|
1754
|
-
if within(root, old).exists():
|
|
1755
|
-
within(root, old).unlink()
|
|
1756
|
-
removed.append(old)
|
|
1757
|
-
return removed
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
def migrate(root, args):
|
|
1761
|
-
config = json.loads(within(root, OLD_CONFIG).read_text())
|
|
1762
|
-
name = config.get("workflow", WORKFLOW)
|
|
1763
|
-
path = f"{WORKFLOW_DIRECTORY}/{name}"
|
|
1764
|
-
text = within(root, path).read_text() if within(root, path).is_file() else ""
|
|
1765
|
-
try:
|
|
1766
|
-
workflow = json.loads(text)
|
|
1767
|
-
job = workflow["jobs"]["replay"]
|
|
1768
|
-
steps = job["steps"]
|
|
1769
|
-
except (ValueError, KeyError, TypeError) as error:
|
|
1770
|
-
if "--cloud-execute" in text and not any(
|
|
1771
|
-
old in text for old in (OLD_SCRIPT, OLD_UPLOAD_STEP, *OLD_RUNNER_ENV)
|
|
1772
|
-
):
|
|
1773
|
-
if config.get("checkCommand") and CHECK_COMMAND_ENV not in text:
|
|
1774
|
-
raise ValueError(
|
|
1775
|
-
f"{OLD_CONFIG} has a checkCommand the workflow lacks. Add {CHECK_COMMAND_ENV}: {shlex.join(config['checkCommand'])} to the Replay step's env, then run --cloud-init again"
|
|
1776
|
-
) from error
|
|
1777
|
-
return {
|
|
1778
|
-
"files": [],
|
|
1779
|
-
"removed": remove_old_files(root),
|
|
1780
|
-
"updated": True,
|
|
1781
|
-
**dropped_command(config),
|
|
1782
|
-
"next": "The workflow already holds these settings, so only the old files were removed",
|
|
1783
|
-
}
|
|
1784
|
-
raise ValueError(
|
|
1785
|
-
f"{path} is not the workflow --cloud-init generated. Make its Replay step run the SDK's bitfab-replay command with --cloud-execute and read each secret in its env, then delete {OLD_CONFIG}"
|
|
1786
|
-
) from error
|
|
1787
|
-
replay = [
|
|
1788
|
-
index
|
|
1789
|
-
for index, step in enumerate(steps)
|
|
1790
|
-
if isinstance(step, dict) and step.get("name") == "Replay"
|
|
1791
|
-
]
|
|
1792
|
-
if len(replay) != 1:
|
|
1793
|
-
raise ValueError(f"{path} must contain exactly one step named Replay")
|
|
1794
|
-
step = steps[replay[0]]
|
|
1795
|
-
words = shlex.split(step.get("run") or "")
|
|
1796
|
-
if args.run:
|
|
1797
|
-
run = args.run
|
|
1798
|
-
elif words[-1:] == ["--cloud-execute"] and OLD_SCRIPT not in step.get("run", ""):
|
|
1799
|
-
run = shlex.join(words[:-1])
|
|
1800
|
-
elif config.get("cliCommand"):
|
|
1801
|
-
run = shlex.join(config["cliCommand"])
|
|
1802
|
-
else:
|
|
1803
|
-
raise ValueError(
|
|
1804
|
-
'The Replay step does not start the SDK\'s bitfab-replay command; pass --run with the command that does, such as --run "npx --no-install bitfab-replay"'
|
|
1805
|
-
)
|
|
1806
|
-
prefix = config.get("secretPrefix", "")
|
|
1807
|
-
env = {
|
|
1808
|
-
**secret_env(config.get("secrets", []), prefix),
|
|
1809
|
-
**{
|
|
1810
|
-
key: value
|
|
1811
|
-
for key, value in (step.get("env") or {}).items()
|
|
1812
|
-
if key not in OLD_RUNNER_ENV
|
|
1813
|
-
},
|
|
1814
|
-
}
|
|
1815
|
-
for variable, source in (config.get("env") or {}).items():
|
|
1816
|
-
kind, target = next(iter(source.items()))
|
|
1817
|
-
env.setdefault(
|
|
1818
|
-
variable, reference("secrets" if kind == "secret" else "vars", target)
|
|
1819
|
-
)
|
|
1820
|
-
if config.get("checkCommand"):
|
|
1821
|
-
env[CHECK_COMMAND_ENV] = shlex.join(config["checkCommand"])
|
|
1822
|
-
kept = {
|
|
1823
|
-
key: value
|
|
1824
|
-
for key, value in step.items()
|
|
1825
|
-
if key not in ("name", "uses", "with", "run", "working-directory", "env")
|
|
1826
|
-
}
|
|
1827
|
-
job["steps"] = [
|
|
1828
|
-
{**replay_step(run, config.get("workingDirectory", "."), env), **kept}
|
|
1829
|
-
if index == replay[0]
|
|
1830
|
-
else entry
|
|
1831
|
-
for index, entry in enumerate(steps)
|
|
1832
|
-
if not (
|
|
1833
|
-
isinstance(entry, dict)
|
|
1834
|
-
and entry.get("name") == OLD_UPLOAD_STEP
|
|
1835
|
-
and str(entry.get("uses", "")).startswith("actions/upload-artifact@")
|
|
1836
|
-
)
|
|
1837
|
-
]
|
|
1838
|
-
if job.get("timeout-minutes") == OLD_JOB_TIMEOUT:
|
|
1839
|
-
del job["timeout-minutes"]
|
|
1840
|
-
replace_file(within(root, path), to_yaml(workflow))
|
|
1841
|
-
result = {
|
|
1842
|
-
"files": [path],
|
|
1843
|
-
"removed": remove_old_files(root),
|
|
1844
|
-
"updated": True,
|
|
1845
|
-
"next": "Review the diff. Replays run the workflow in their own snapshot, so the change applies without merging",
|
|
1762
|
+
"secrets": secrets,
|
|
1763
|
+
"reviewPushTriggers": triggers,
|
|
1764
|
+
"next": [
|
|
1765
|
+
"Compare the install steps with your CI and add anything detection missed, such as a build or code-generation step",
|
|
1766
|
+
f"Create the secrets {', '.join(secrets)} at {where}, or copy them from a local file with bitfab-replay --cloud-secrets --env-file .env",
|
|
1767
|
+
*(
|
|
1768
|
+
[
|
|
1769
|
+
f"Keep bitfab-replay/** branches from triggering {', '.join(triggers)}"
|
|
1770
|
+
]
|
|
1771
|
+
if triggers
|
|
1772
|
+
else []
|
|
1773
|
+
),
|
|
1774
|
+
f"Commit {path} and merge it to the default branch once, so GitHub registers it",
|
|
1775
|
+
"Add --cloud --dry-run to your replay command to check the runner, then replay with --cloud",
|
|
1776
|
+
],
|
|
1846
1777
|
}
|
|
1847
|
-
return {**result, **dropped_command(config)}
|
|
1848
1778
|
|
|
1849
1779
|
|
|
1850
|
-
def
|
|
1851
|
-
|
|
1852
|
-
return {}
|
|
1853
|
-
return {
|
|
1854
|
-
"droppedCommand": config["command"],
|
|
1855
|
-
"droppedCommandNext": "The runner now starts the same bitfab-replay you run locally, from the same directory and with the same arguments, so this command is no longer used. "
|
|
1856
|
-
"If it also set environment variables, add them to the Replay step's env",
|
|
1857
|
-
}
|
|
1780
|
+
def print_json(value):
|
|
1781
|
+
print(json.dumps(value, indent=2), flush=True)
|
|
1858
1782
|
|
|
1859
1783
|
|
|
1860
1784
|
def main():
|
|
1861
1785
|
argv = sys.argv[1:]
|
|
1862
1786
|
try:
|
|
1863
1787
|
if argv[:1] == ["--cloud-init"]:
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1788
|
+
print_json(initialize(argv[1:]))
|
|
1789
|
+
return 0
|
|
1790
|
+
if argv[:1] == ["--cloud-secrets"]:
|
|
1791
|
+
print_json(configure_secrets(argv[1:]))
|
|
1792
|
+
return 0
|
|
1793
|
+
if argv == ["--cloud-execute"]:
|
|
1868
1794
|
result = execute()
|
|
1869
|
-
|
|
1795
|
+
return result["exitCode"] or (1 if result.get("stoppedEarly") else 0)
|
|
1796
|
+
if "-h" in argv or "--help" in argv:
|
|
1870
1797
|
print(HELP, end="")
|
|
1871
1798
|
return 0
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
print(
|
|
1878
|
-
f"Cloud check failed. The Replay step log names what is missing: {result.get('url')}",
|
|
1879
|
-
file=sys.stderr,
|
|
1880
|
-
)
|
|
1881
|
-
elif result.get("testRunId"):
|
|
1882
|
-
print(
|
|
1883
|
-
f"Cloud replay stopped early ({result.get('stoppedEarly', result.get('conclusion'))}). Traces that finished are saved in test run {result['testRunId']}.",
|
|
1884
|
-
file=sys.stderr,
|
|
1885
|
-
)
|
|
1886
|
-
return 1
|
|
1887
|
-
if result.get("state") == "completed":
|
|
1888
|
-
code = report_errored_items(result)
|
|
1889
|
-
if result.get("failOnError") and result.get("errored"):
|
|
1890
|
-
print(
|
|
1891
|
-
"Cloud replay: exiting 1 because of --fail-on-error",
|
|
1892
|
-
file=sys.stderr,
|
|
1893
|
-
)
|
|
1894
|
-
return 1
|
|
1895
|
-
return code
|
|
1896
|
-
return 0
|
|
1799
|
+
record, found = run_cli(argv)
|
|
1800
|
+
if found is not None:
|
|
1801
|
+
return print_outcome(found)
|
|
1802
|
+
print_json(record)
|
|
1803
|
+
return record.get("exitCode", 0) if record.get("state") == "completed" else 0
|
|
1897
1804
|
except KeyboardInterrupt:
|
|
1898
|
-
|
|
1899
|
-
"Detached. The GitHub
|
|
1900
|
-
file=sys.stderr,
|
|
1805
|
+
progress(
|
|
1806
|
+
"Detached. The GitHub run continues; follow it with the printed execution UUID"
|
|
1901
1807
|
)
|
|
1902
1808
|
return 130
|
|
1903
1809
|
except (
|