bitfab 0.63.3 → 0.64.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/cloudReplay.py +664 -777
- data/lib/bitfab/cloud_replay_cli.rb +10 -43
- data/lib/bitfab/replay_cli.rb +2 -2
- data/lib/bitfab/version.rb +1 -1
- metadata +1 -1
data/lib/bitfab/cloudReplay.py
CHANGED
|
@@ -19,6 +19,7 @@ import time
|
|
|
19
19
|
import urllib.error
|
|
20
20
|
import urllib.request
|
|
21
21
|
import uuid
|
|
22
|
+
import zlib
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
from urllib.parse import urlencode
|
|
24
25
|
|
|
@@ -26,19 +27,21 @@ WORKFLOW = "bitfab-replay.yml"
|
|
|
26
27
|
WORKFLOW_NAMES = (WORKFLOW, "bitfab-replay.yaml")
|
|
27
28
|
WORKFLOW_DIRECTORY = ".github/workflows"
|
|
28
29
|
OLD_CONFIG = ".bitfab/cloud.json"
|
|
29
|
-
OLD_SCRIPT = ".bitfab/cloudReplay.py"
|
|
30
|
-
OLD_RUNNER_ENV = ("BITFAB_CLOUD_REQUEST", "BITFAB_EXECUTION_ID", "BITFAB_COMMIT_SHA")
|
|
31
|
-
OLD_UPLOAD_STEP = "Save replay identity"
|
|
32
|
-
OLD_JOB_TIMEOUT = 35
|
|
33
30
|
PREFIX = "bitfab-replay/"
|
|
34
31
|
DEFAULT_SECRET_PREFIX = "BITFAB_CLOUD_"
|
|
35
32
|
API_VERSION = "2026-03-10"
|
|
36
|
-
RUN_NAME = "Bitfab replay ${{ inputs.execution_id }}"
|
|
37
|
-
RESULT_TITLE = "Bitfab replay result"
|
|
38
33
|
REPLAY_COMMAND_ENV = "BITFAB_REPLAY_COMMAND"
|
|
39
34
|
SDK_LANGUAGE_ENV = "BITFAB_SDK_LANGUAGE"
|
|
40
35
|
CHECK_COMMAND_ENV = "BITFAB_REPLAY_CHECK"
|
|
41
|
-
REQUEST_VERSION =
|
|
36
|
+
REQUEST_VERSION = 3
|
|
37
|
+
RESULT_LINE = "bitfab-replay-result "
|
|
38
|
+
RESULT_CHUNK = 4000
|
|
39
|
+
OUTPUT_BEGIN = "bitfab-replay-output-begin"
|
|
40
|
+
OUTPUT_END = "bitfab-replay-output-end"
|
|
41
|
+
LOG_TAIL_LINES = 40
|
|
42
|
+
OUTPUT_LIMIT = 16 * 1024 * 1024
|
|
43
|
+
NEW_FILES_LIMIT = 20 * 1024 * 1024
|
|
44
|
+
POLL_SECONDS = 5
|
|
42
45
|
ITEM_ERROR_FIELDS = (
|
|
43
46
|
"error",
|
|
44
47
|
"traceError",
|
|
@@ -49,10 +52,12 @@ ITEM_ERROR_FIELDS = (
|
|
|
49
52
|
ITEM_ID_FIELDS = ("originalTraceId", "original_trace_id", "traceId", "trace_id")
|
|
50
53
|
ITEM_ERROR_LINES = 20
|
|
51
54
|
ITEM_ERROR_LENGTH = 500
|
|
52
|
-
OUTPUT_TAIL_LENGTH = 4000
|
|
53
55
|
UUID = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$")
|
|
54
56
|
SHA = re.compile(r"^[0-9a-f]{40}$")
|
|
55
|
-
|
|
57
|
+
GITHUB_REMOTE = re.compile(
|
|
58
|
+
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?"
|
|
59
|
+
)
|
|
60
|
+
LOG_TIMESTAMP = re.compile(r"^?\d{4}-\d\d-\d\dT[\d:.]+Z ?")
|
|
56
61
|
EXPERIMENT_LINE = re.compile(rb"^\[replay\] Experiment ([0-9a-f-]{36}):")
|
|
57
62
|
SECRET_REFERENCE = re.compile(
|
|
58
63
|
r"[\"']?([A-Za-z_][A-Za-z0-9_]*)[\"']?[ \t]*:[ \t]*[\"']?\$\{\{\s*secrets(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[\s*[\"']([A-Za-z_][A-Za-z0-9_]*)[\"']\s*\])\s*\}\}"
|
|
@@ -64,27 +69,20 @@ ENVIRONMENT_SETTING = re.compile(
|
|
|
64
69
|
PUSH_TRIGGER = re.compile(
|
|
65
70
|
r"^[ \t]*\"?(?:on\"?[ \t]*:.*\bpush\b|push\"?[ \t]*:)", re.MULTILINE
|
|
66
71
|
)
|
|
67
|
-
|
|
72
|
+
DECLARED_INPUTS = re.compile(r"^[ \t]*\"?execution_id\"?[ \t]*:", re.MULTILINE)
|
|
73
|
+
FOLLOW_FLAGS = (
|
|
68
74
|
"--cloud-status",
|
|
69
75
|
"--cloud-watch",
|
|
70
76
|
"--cloud-cancel",
|
|
71
77
|
"--cloud-cleanup",
|
|
72
78
|
)
|
|
73
|
-
CLOUD_VALUE_FLAGS = (
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
"--cloud-
|
|
77
|
-
"--cloud-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
"--cloud",
|
|
81
|
-
"--cloud-dry-run",
|
|
82
|
-
"--cloud-detach",
|
|
83
|
-
"--cloud-check",
|
|
84
|
-
"--fail-on-error",
|
|
85
|
-
"--dry-run",
|
|
86
|
-
)
|
|
87
|
-
CHECK_SWITCHES = ("--cloud-check", "--dry-run")
|
|
79
|
+
CLOUD_VALUE_FLAGS = (*FOLLOW_FLAGS, "--cloud-request-id", "--cloud-timeout")
|
|
80
|
+
CLOUD_SWITCHES = ("--cloud", "--cloud-preview", "--cloud-detach")
|
|
81
|
+
RENAMED_FLAGS = {
|
|
82
|
+
"--cloud-dry-run": "--cloud-preview",
|
|
83
|
+
"--cloud-check": "--dry-run",
|
|
84
|
+
"--cloud-include": "nothing: new files that git does not ignore are already in the snapshot",
|
|
85
|
+
}
|
|
88
86
|
SEED_FLAGS = ("--seed", "--cases", "--from-trace", "--run")
|
|
89
87
|
PATH_FLAGS = ("--registry", "--params", "--code-change")
|
|
90
88
|
CODE_CHANGE_FLAGS = ("--code-change", "--no-code-change")
|
|
@@ -109,36 +107,32 @@ HELP = """Cloud replay: run the replay you run locally on GitHub Actions by addi
|
|
|
109
107
|
|
|
110
108
|
bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud
|
|
111
109
|
|
|
112
|
-
Every replay option works as it does locally
|
|
113
|
-
The runner
|
|
114
|
-
|
|
115
|
-
|
|
110
|
+
Every replay option works as it does locally, including --dry-run and --fail-on-error.
|
|
111
|
+
The runner replays a snapshot of your working tree (changed files, plus new files git
|
|
112
|
+
does not ignore) from the same directory, and this command prints the replay's output
|
|
113
|
+
and exits with its exit code. Credential-like files are refused. Requires git and
|
|
114
|
+
Python 3.10+ on macOS or Linux, with a github.com origin.
|
|
116
115
|
|
|
117
116
|
Options added by --cloud:
|
|
118
|
-
--
|
|
119
|
-
maps has a value, BITFAB_REPLAY_CHECK passes when set, and the
|
|
120
|
-
traces resolve. --cloud-check is the same.
|
|
121
|
-
--cloud-dry-run Show what the snapshot would contain and push nothing.
|
|
122
|
-
--cloud-include PATH Add a new file to the snapshot (repeatable).
|
|
117
|
+
--cloud-preview List what the snapshot would contain and push nothing.
|
|
123
118
|
--cloud-detach Return after dispatch instead of waiting.
|
|
124
119
|
--cloud-timeout MIN Stop the replay after MIN minutes (1..7200).
|
|
125
120
|
--cloud-request-id ID Recover a submission whose response was lost.
|
|
126
|
-
--fail-on-error Exit 1 when any replayed item errored.
|
|
127
121
|
|
|
128
122
|
Follow a replay by the execution UUID it prints:
|
|
129
|
-
--cloud-
|
|
123
|
+
--cloud-watch ID | --cloud-status ID | --cloud-cancel ID | --cloud-cleanup ID
|
|
130
124
|
|
|
131
125
|
Set up once:
|
|
132
126
|
--cloud-init [--secret NAME ...] [--environment NAME] [--run COMMAND]
|
|
133
127
|
[--runs-on LABEL] [--secret-prefix PREFIX] [--check COMMAND]
|
|
134
128
|
Writes .github/workflows/bitfab-replay.yml, the only file cloud replay keeps
|
|
135
|
-
in the repository
|
|
129
|
+
in the repository, and lists what is left to do.
|
|
136
130
|
--cloud-secrets --env-file FILE [NAME ...] [--environment NAME] [--dry-run]
|
|
137
|
-
|
|
131
|
+
Copies local values into the GitHub secrets the workflow reads.
|
|
138
132
|
|
|
139
133
|
GitHub access comes from the GitHub CLI when it is logged in, otherwise from GH_TOKEN
|
|
140
|
-
or GITHUB_TOKEN, otherwise from the github.com credential git already stores.
|
|
141
|
-
|
|
134
|
+
or GITHUB_TOKEN, otherwise from the github.com credential git already stores. Without
|
|
135
|
+
the GitHub CLI, --cloud-secrets lists the secrets to create by hand and where.
|
|
142
136
|
"""
|
|
143
137
|
|
|
144
138
|
|
|
@@ -148,6 +142,11 @@ class CommandError(RuntimeError):
|
|
|
148
142
|
self.http_status = http_status
|
|
149
143
|
|
|
150
144
|
|
|
145
|
+
class KeepRedirect(urllib.request.HTTPRedirectHandler):
|
|
146
|
+
def redirect_request(self, *args, **kwargs):
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
151
150
|
def command(args, *, cwd=None, env=None, timeout=60, input=None):
|
|
152
151
|
result = subprocess.run(
|
|
153
152
|
args,
|
|
@@ -175,6 +174,10 @@ def git(root, *args, env=None, input=None):
|
|
|
175
174
|
return command(["git", "-C", str(root), *args], env=env, input=input)
|
|
176
175
|
|
|
177
176
|
|
|
177
|
+
def progress(message):
|
|
178
|
+
print(f"[cloud] {message}", file=sys.stderr, flush=True)
|
|
179
|
+
|
|
180
|
+
|
|
178
181
|
def root_directory():
|
|
179
182
|
return Path(command(["git", "rev-parse", "--show-toplevel"])).resolve()
|
|
180
183
|
|
|
@@ -195,12 +198,9 @@ def within(root, value):
|
|
|
195
198
|
|
|
196
199
|
def repository_path(root, path):
|
|
197
200
|
try:
|
|
198
|
-
|
|
201
|
+
return Path(path).resolve().relative_to(root).as_posix()
|
|
199
202
|
except ValueError as error:
|
|
200
203
|
raise ValueError(f"{path} must be inside the repository") from error
|
|
201
|
-
if sensitive(relative):
|
|
202
|
-
raise ValueError(f"Refusing credential-like file: {relative}")
|
|
203
|
-
return relative
|
|
204
204
|
|
|
205
205
|
|
|
206
206
|
def working_directory(root):
|
|
@@ -212,21 +212,15 @@ def working_directory(root):
|
|
|
212
212
|
|
|
213
213
|
|
|
214
214
|
def repository(root):
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?", remote
|
|
218
|
-
)
|
|
219
|
-
if match is None:
|
|
215
|
+
fetch = GITHUB_REMOTE.fullmatch(git(root, "remote", "get-url", "origin"))
|
|
216
|
+
if fetch is None:
|
|
220
217
|
raise ValueError(
|
|
221
218
|
"origin must be a github.com repository without embedded credentials"
|
|
222
219
|
)
|
|
223
|
-
push = git(root, "remote", "get-url", "--push", "origin")
|
|
224
|
-
|
|
225
|
-
r"(?:git@github\.com:|https://github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?", push
|
|
226
|
-
)
|
|
227
|
-
if other is None or other[1].lower() != match[1].lower():
|
|
220
|
+
push = GITHUB_REMOTE.fullmatch(git(root, "remote", "get-url", "--push", "origin"))
|
|
221
|
+
if push is None or push[1].lower() != fetch[1].lower():
|
|
228
222
|
raise ValueError("origin fetch and push repositories must match")
|
|
229
|
-
return
|
|
223
|
+
return fetch[1]
|
|
230
224
|
|
|
231
225
|
|
|
232
226
|
def find_workflow(root):
|
|
@@ -236,11 +230,15 @@ def find_workflow(root):
|
|
|
236
230
|
return None
|
|
237
231
|
|
|
238
232
|
|
|
239
|
-
def
|
|
233
|
+
def refuse_old_setup(root):
|
|
240
234
|
if within(root, OLD_CONFIG).exists():
|
|
241
235
|
raise ValueError(
|
|
242
|
-
f"This cloud replay setup was made by an older SDK.
|
|
236
|
+
f"This cloud replay setup was made by an older SDK. Delete {OLD_CONFIG} and {WORKFLOW_DIRECTORY}/{WORKFLOW}, then run bitfab-replay --cloud-init"
|
|
243
237
|
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def setup_workflow(root):
|
|
241
|
+
refuse_old_setup(root)
|
|
244
242
|
name = find_workflow(root)
|
|
245
243
|
if name is None:
|
|
246
244
|
raise ValueError(
|
|
@@ -256,6 +254,12 @@ def workflow_secrets(text):
|
|
|
256
254
|
}
|
|
257
255
|
|
|
258
256
|
|
|
257
|
+
def secrets_page(repo, environment):
|
|
258
|
+
if environment:
|
|
259
|
+
return f"https://github.com/{repo}/settings/environments"
|
|
260
|
+
return f"https://github.com/{repo}/settings/secrets/actions"
|
|
261
|
+
|
|
262
|
+
|
|
259
263
|
ENV_ASSIGNMENT = re.compile(r"[ \t]*(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*=")
|
|
260
264
|
|
|
261
265
|
ENV_ESCAPES = {"n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"}
|
|
@@ -306,10 +310,26 @@ def parse_environment_file(text):
|
|
|
306
310
|
return values
|
|
307
311
|
|
|
308
312
|
|
|
313
|
+
def ensure_environment(repo, environment):
|
|
314
|
+
try:
|
|
315
|
+
api(repo, f"environments/{environment}")
|
|
316
|
+
return "exists"
|
|
317
|
+
except ValueError:
|
|
318
|
+
return "unchecked"
|
|
319
|
+
except RuntimeError as error:
|
|
320
|
+
if getattr(error, "http_status", None) != 404:
|
|
321
|
+
return "unchecked"
|
|
322
|
+
try:
|
|
323
|
+
api(repo, f"environments/{environment}", method="PUT", payload={})
|
|
324
|
+
return "created"
|
|
325
|
+
except RuntimeError:
|
|
326
|
+
return "missing"
|
|
327
|
+
|
|
328
|
+
|
|
309
329
|
def configure_secrets(argv):
|
|
310
330
|
parser = argparse.ArgumentParser(
|
|
311
331
|
prog="bitfab-replay --cloud-secrets",
|
|
312
|
-
description="Copy named values from local environment files into the GitHub Actions secrets the cloud replay workflow reads. Values are piped to
|
|
332
|
+
description="Copy named values from local environment files into the GitHub Actions secrets the cloud replay workflow reads. Values are piped to the GitHub CLI on standard input and never printed, logged, or passed as arguments. Without the GitHub CLI, it lists the secrets to create by hand and where.",
|
|
313
333
|
)
|
|
314
334
|
parser.add_argument(
|
|
315
335
|
"names",
|
|
@@ -324,7 +344,7 @@ def configure_secrets(argv):
|
|
|
324
344
|
)
|
|
325
345
|
parser.add_argument(
|
|
326
346
|
"--environment",
|
|
327
|
-
help="GitHub Environment to set the secrets in; defaults to the workflow's environment",
|
|
347
|
+
help="GitHub Environment to set the secrets in; defaults to the workflow's environment, and is created when it does not exist",
|
|
328
348
|
)
|
|
329
349
|
parser.add_argument(
|
|
330
350
|
"--dry-run",
|
|
@@ -333,14 +353,13 @@ def configure_secrets(argv):
|
|
|
333
353
|
)
|
|
334
354
|
args = parser.parse_args(argv)
|
|
335
355
|
root = root_directory()
|
|
336
|
-
|
|
337
|
-
text = within(root, f"{WORKFLOW_DIRECTORY}/{workflow}").read_text()
|
|
356
|
+
text = within(root, f"{WORKFLOW_DIRECTORY}/{setup_workflow(root)}").read_text()
|
|
338
357
|
targets = workflow_secrets(text)
|
|
339
358
|
names = args.names or list(targets)
|
|
340
359
|
unknown = [name for name in names if name not in targets]
|
|
341
360
|
if unknown:
|
|
342
361
|
raise ValueError(
|
|
343
|
-
f"The workflow does not read {', '.join(unknown)}. Add each one to the
|
|
362
|
+
f"The workflow does not read {', '.join(unknown)}. Add each one to the replay step's env as NAME: ${{{{ secrets.{DEFAULT_SECRET_PREFIX}NAME }}}} first"
|
|
344
363
|
)
|
|
345
364
|
configured = ENVIRONMENT_SETTING.search(text)
|
|
346
365
|
environment = args.environment or (configured[1] if configured else None)
|
|
@@ -355,51 +374,60 @@ def configure_secrets(argv):
|
|
|
355
374
|
for name in names:
|
|
356
375
|
value = values.get(name, values.get(targets[name]))
|
|
357
376
|
if value is None:
|
|
358
|
-
missing.append(name)
|
|
377
|
+
missing.append(targets[name])
|
|
359
378
|
elif not value:
|
|
360
|
-
empty.append(name)
|
|
379
|
+
empty.append(targets[name])
|
|
361
380
|
else:
|
|
362
381
|
found.append(name)
|
|
363
|
-
|
|
364
|
-
where = (
|
|
365
|
-
f"https://github.com/{repo}/settings/environments"
|
|
366
|
-
if environment
|
|
367
|
-
else f"https://github.com/{repo}/settings/secrets/actions"
|
|
368
|
-
)
|
|
369
|
-
raise ValueError(
|
|
370
|
-
f"Copying secrets needs the GitHub CLI, which encrypts each value for GitHub. Install it (https://cli.github.com), or create {', '.join(targets[name] for name in found)} by hand at {where}"
|
|
371
|
-
)
|
|
372
|
-
for name in found:
|
|
373
|
-
if not args.dry_run:
|
|
374
|
-
command(
|
|
375
|
-
[
|
|
376
|
-
"gh",
|
|
377
|
-
"secret",
|
|
378
|
-
"set",
|
|
379
|
-
targets[name],
|
|
380
|
-
"--repo",
|
|
381
|
-
repo,
|
|
382
|
-
*(["--env", environment] if environment else []),
|
|
383
|
-
],
|
|
384
|
-
input=values.get(name, values.get(targets[name])),
|
|
385
|
-
)
|
|
386
|
-
return {
|
|
382
|
+
result = {
|
|
387
383
|
"repository": repo,
|
|
388
384
|
"environment": environment,
|
|
389
385
|
"dryRun": args.dry_run,
|
|
390
|
-
"set": [
|
|
386
|
+
"set": [],
|
|
391
387
|
"missing": missing,
|
|
392
388
|
"empty": empty,
|
|
393
|
-
"next": "Create the missing secrets by hand; empty local values were skipped because an empty secret overrides a working default with nothing; a wrong value only surfaces when the first real replay runs",
|
|
394
389
|
}
|
|
390
|
+
if args.dry_run:
|
|
391
|
+
return {**result, "wouldSet": [targets[name] for name in found]}
|
|
392
|
+
if environment:
|
|
393
|
+
result["environmentStatus"] = ensure_environment(repo, environment)
|
|
394
|
+
if found and shutil.which("gh") is None:
|
|
395
|
+
progress(
|
|
396
|
+
"Setting secrets needs the GitHub CLI (https://cli.github.com), which encrypts each value for GitHub; create the ones listed under createByHand instead"
|
|
397
|
+
)
|
|
398
|
+
return {
|
|
399
|
+
**result,
|
|
400
|
+
"createByHand": [targets[name] for name in found],
|
|
401
|
+
"where": secrets_page(repo, environment),
|
|
402
|
+
}
|
|
403
|
+
token = github_access()["token"]
|
|
404
|
+
for name in found:
|
|
405
|
+
command(
|
|
406
|
+
[
|
|
407
|
+
"gh",
|
|
408
|
+
"secret",
|
|
409
|
+
"set",
|
|
410
|
+
targets[name],
|
|
411
|
+
"--repo",
|
|
412
|
+
repo,
|
|
413
|
+
*(["--env", environment] if environment else []),
|
|
414
|
+
],
|
|
415
|
+
input=values.get(name, values.get(targets[name])),
|
|
416
|
+
env={**os.environ, "GH_TOKEN": token},
|
|
417
|
+
)
|
|
418
|
+
result["set"] = [targets[name] for name in found]
|
|
419
|
+
if missing or empty:
|
|
420
|
+
result["where"] = secrets_page(repo, environment)
|
|
421
|
+
return result
|
|
395
422
|
|
|
396
423
|
|
|
397
424
|
@functools.cache
|
|
398
425
|
def github_access():
|
|
399
426
|
if shutil.which("gh") is not None:
|
|
400
427
|
with contextlib.suppress(RuntimeError, OSError, subprocess.SubprocessError):
|
|
401
|
-
command(["gh", "auth", "
|
|
402
|
-
|
|
428
|
+
token = command(["gh", "auth", "token", "--hostname", "github.com"])
|
|
429
|
+
if token:
|
|
430
|
+
return {"source": "the GitHub CLI", "token": token}
|
|
403
431
|
for name in ("GH_TOKEN", "GITHUB_TOKEN"):
|
|
404
432
|
if os.environ.get(name):
|
|
405
433
|
return {"source": name, "token": os.environ[name]}
|
|
@@ -436,29 +464,10 @@ def stored_git_credential():
|
|
|
436
464
|
return None
|
|
437
465
|
|
|
438
466
|
|
|
439
|
-
def api(repo, suffix, *, method="GET", payload=None):
|
|
467
|
+
def api(repo, suffix, *, method="GET", payload=None, raw=False):
|
|
440
468
|
path = f"repos/{repo}" + (f"/{suffix}" if suffix else "")
|
|
441
469
|
access = github_access()
|
|
442
470
|
data = None if payload is None else json.dumps(payload)
|
|
443
|
-
if access["token"] is None:
|
|
444
|
-
result = command(
|
|
445
|
-
[
|
|
446
|
-
"gh",
|
|
447
|
-
"api",
|
|
448
|
-
"--hostname",
|
|
449
|
-
"github.com",
|
|
450
|
-
"-H",
|
|
451
|
-
f"X-GitHub-Api-Version: {API_VERSION}",
|
|
452
|
-
"--jq",
|
|
453
|
-
"tojson",
|
|
454
|
-
path,
|
|
455
|
-
"--method",
|
|
456
|
-
method,
|
|
457
|
-
*(["--input", "-"] if data is not None else []),
|
|
458
|
-
],
|
|
459
|
-
input=data,
|
|
460
|
-
)
|
|
461
|
-
return json.loads(result) if result else None
|
|
462
471
|
request = urllib.request.Request(
|
|
463
472
|
f"https://api.github.com/{path}",
|
|
464
473
|
method=method,
|
|
@@ -472,8 +481,7 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
472
481
|
},
|
|
473
482
|
)
|
|
474
483
|
try:
|
|
475
|
-
|
|
476
|
-
body = response.read()
|
|
484
|
+
body = fetch(request, follow_without_credentials=raw)
|
|
477
485
|
except urllib.error.HTTPError as error:
|
|
478
486
|
raise CommandError(
|
|
479
487
|
f"GitHub {method} {path.split('?')[0]} failed (HTTP {error.code}) using {access['source']}",
|
|
@@ -485,18 +493,43 @@ def api(repo, suffix, *, method="GET", payload=None):
|
|
|
485
493
|
raise CommandError(
|
|
486
494
|
"Python cannot verify GitHub's certificate because it has no certificate store. "
|
|
487
495
|
"On a python.org install, run Install Certificates.command from its Applications folder, "
|
|
488
|
-
"set SSL_CERT_FILE to a certificate bundle
|
|
496
|
+
"or set SSL_CERT_FILE to a certificate bundle"
|
|
489
497
|
) from None
|
|
490
498
|
raise CommandError(f"Could not reach api.github.com: {reason}") from None
|
|
499
|
+
if raw:
|
|
500
|
+
return body.decode(errors="replace").strip()
|
|
491
501
|
return json.loads(body) if body.strip() else None
|
|
492
502
|
|
|
493
503
|
|
|
504
|
+
def fetch(request, *, follow_without_credentials):
|
|
505
|
+
if not follow_without_credentials:
|
|
506
|
+
with urllib.request.urlopen(request, timeout=60) as response:
|
|
507
|
+
return response.read()
|
|
508
|
+
try:
|
|
509
|
+
with urllib.request.build_opener(KeepRedirect).open(
|
|
510
|
+
request, timeout=60
|
|
511
|
+
) as response:
|
|
512
|
+
return response.read()
|
|
513
|
+
except urllib.error.HTTPError as error:
|
|
514
|
+
location = error.headers.get("Location")
|
|
515
|
+
if error.code not in (301, 302, 303, 307, 308) or not location:
|
|
516
|
+
raise
|
|
517
|
+
with urllib.request.urlopen(location, timeout=120) as response:
|
|
518
|
+
return response.read()
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def option_name(token):
|
|
522
|
+
return token.split("=", 1)[0]
|
|
523
|
+
|
|
524
|
+
|
|
494
525
|
def split_arguments(argv):
|
|
495
|
-
cloud,
|
|
526
|
+
cloud, switches, replay = {}, set(), []
|
|
496
527
|
index = 0
|
|
497
528
|
while index < len(argv):
|
|
498
529
|
token = argv[index]
|
|
499
|
-
name = token
|
|
530
|
+
name = option_name(token)
|
|
531
|
+
if name in RENAMED_FLAGS:
|
|
532
|
+
raise ValueError(f"{name} is gone; use {RENAMED_FLAGS[name]}")
|
|
500
533
|
if name in CLOUD_VALUE_FLAGS:
|
|
501
534
|
if "=" in token:
|
|
502
535
|
value = token.split("=", 1)[1]
|
|
@@ -506,12 +539,9 @@ def split_arguments(argv):
|
|
|
506
539
|
index += 2
|
|
507
540
|
else:
|
|
508
541
|
raise ValueError(f"{name} needs a value")
|
|
509
|
-
if name
|
|
510
|
-
includes.append(value)
|
|
511
|
-
elif name in cloud:
|
|
542
|
+
if name in cloud:
|
|
512
543
|
raise ValueError(f"{name} was given twice")
|
|
513
|
-
|
|
514
|
-
cloud[name] = value
|
|
544
|
+
cloud[name] = value
|
|
515
545
|
elif token in CLOUD_SWITCHES:
|
|
516
546
|
switches.add(token)
|
|
517
547
|
index += 1
|
|
@@ -520,23 +550,23 @@ def split_arguments(argv):
|
|
|
520
550
|
else:
|
|
521
551
|
replay.append(token)
|
|
522
552
|
index += 1
|
|
523
|
-
return cloud,
|
|
553
|
+
return cloud, switches, replay
|
|
524
554
|
|
|
525
555
|
|
|
526
556
|
def parse(argv):
|
|
527
|
-
cloud,
|
|
528
|
-
|
|
529
|
-
if
|
|
530
|
-
if len(
|
|
531
|
-
raise ValueError("Cloud
|
|
532
|
-
operation =
|
|
533
|
-
execution_id = cloud[
|
|
557
|
+
cloud, switches, replay = split_arguments(argv)
|
|
558
|
+
following = [flag for flag in FOLLOW_FLAGS if flag in cloud]
|
|
559
|
+
if following:
|
|
560
|
+
if len(following) != 1 or len(cloud) != 1 or switches - {"--cloud"} or replay:
|
|
561
|
+
raise ValueError("Cloud follow-up commands take only their execution UUID")
|
|
562
|
+
operation = following[0].removeprefix("--cloud-")
|
|
563
|
+
execution_id = cloud[following[0]]
|
|
534
564
|
else:
|
|
535
565
|
if not replay:
|
|
536
566
|
raise ValueError(
|
|
537
567
|
"Add --cloud to the replay command you run locally, such as bitfab-replay --registry scripts/replay.ts classify --trace-ids UUID --cloud"
|
|
538
568
|
)
|
|
539
|
-
if not any(token
|
|
569
|
+
if not any(option_name(token) in SELECTION_FLAGS for token in replay):
|
|
540
570
|
raise ValueError(
|
|
541
571
|
"Select traces with --trace-ids, --dataset-ids, or --resume"
|
|
542
572
|
)
|
|
@@ -553,7 +583,6 @@ def parse(argv):
|
|
|
553
583
|
"operation": operation,
|
|
554
584
|
"id": execution_id,
|
|
555
585
|
"cloud": cloud,
|
|
556
|
-
"includes": includes,
|
|
557
586
|
"switches": switches,
|
|
558
587
|
"replay": replay,
|
|
559
588
|
}
|
|
@@ -573,6 +602,10 @@ def validate_arguments(args):
|
|
|
573
602
|
raise ValueError("Replay options exceed 16000 characters")
|
|
574
603
|
|
|
575
604
|
|
|
605
|
+
def dry_run(args):
|
|
606
|
+
return "--dry-run" in args
|
|
607
|
+
|
|
608
|
+
|
|
576
609
|
def replay_request(root, parsed):
|
|
577
610
|
cwd = Path.cwd().resolve()
|
|
578
611
|
args, files = [], []
|
|
@@ -606,8 +639,6 @@ def replay_request(root, parsed):
|
|
|
606
639
|
}
|
|
607
640
|
if "--cloud-timeout" in parsed["cloud"]:
|
|
608
641
|
request["timeoutMinutes"] = int(parsed["cloud"]["--cloud-timeout"])
|
|
609
|
-
if parsed["switches"] & set(CHECK_SWITCHES):
|
|
610
|
-
request["check"] = True
|
|
611
642
|
return request, files
|
|
612
643
|
|
|
613
644
|
|
|
@@ -634,81 +665,71 @@ def sensitive(path):
|
|
|
634
665
|
)
|
|
635
666
|
|
|
636
667
|
|
|
637
|
-
def snapshot(root, workflow,
|
|
668
|
+
def snapshot(root, workflow, files, request, *, preview=False):
|
|
638
669
|
if git(root, "ls-files", "-u"):
|
|
639
670
|
raise ValueError("Resolve merge conflicts before snapshotting")
|
|
640
671
|
head = git(root, "rev-parse", "HEAD")
|
|
641
|
-
included = []
|
|
642
|
-
for value in includes:
|
|
643
|
-
path = within(root, value)
|
|
644
|
-
if not path.is_file() or path.is_symlink() or (root / value).is_symlink():
|
|
645
|
-
raise ValueError("--cloud-include requires individual regular files")
|
|
646
|
-
ignored = subprocess.run(
|
|
647
|
-
["git", "-C", str(root), "check-ignore", "--quiet", "--", value],
|
|
648
|
-
check=False,
|
|
649
|
-
timeout=15,
|
|
650
|
-
).returncode
|
|
651
|
-
if ignored not in (0, 1):
|
|
652
|
-
raise ValueError("Unable to determine whether the included file is ignored")
|
|
653
|
-
if ignored == 0 or sensitive(value):
|
|
654
|
-
raise ValueError(f"Refusing ignored or credential-like file: {value}")
|
|
655
|
-
included.append(value)
|
|
656
672
|
with tempfile.TemporaryDirectory(prefix="bitfab-cloud-index-") as directory:
|
|
657
673
|
env = {**os.environ, "GIT_INDEX_FILE": str(Path(directory) / "index")}
|
|
658
674
|
git(root, "read-tree", head, env=env)
|
|
659
|
-
git(root, "add", "-
|
|
660
|
-
if included:
|
|
661
|
-
git(root, "--literal-pathspecs", "add", "--", *included, env=env)
|
|
675
|
+
git(root, "add", "-A", "--", ".", env=env)
|
|
662
676
|
entries = git(root, "ls-files", "--stage", "-z", env=env).split("\x00")
|
|
677
|
+
tracked = set()
|
|
663
678
|
for entry in filter(None, entries):
|
|
664
679
|
metadata, path = entry.split("\t", 1)
|
|
680
|
+
tracked.add(path)
|
|
665
681
|
if metadata.startswith("160000"):
|
|
666
682
|
raise ValueError(
|
|
667
|
-
"
|
|
683
|
+
f"{path} is a submodule or nested repository, which cloud snapshots do not support"
|
|
668
684
|
)
|
|
669
685
|
if sensitive(path):
|
|
670
686
|
raise ValueError(
|
|
671
|
-
f"Refusing credential-like
|
|
687
|
+
f"Refusing credential-like file {path}; add it to .gitignore, or git rm --cached it when it is tracked"
|
|
672
688
|
)
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
untracked = [path for path in required if path not in tracked]
|
|
680
|
-
if untracked:
|
|
689
|
+
ignored = [
|
|
690
|
+
path
|
|
691
|
+
for path in [f"{WORKFLOW_DIRECTORY}/{workflow}", *files]
|
|
692
|
+
if path not in tracked
|
|
693
|
+
]
|
|
694
|
+
if ignored:
|
|
681
695
|
raise ValueError(
|
|
682
|
-
"
|
|
683
|
-
+ ", ".join(
|
|
684
|
-
+ "
|
|
696
|
+
"Git ignores "
|
|
697
|
+
+ ", ".join(ignored)
|
|
698
|
+
+ ", so the snapshot cannot include it; stop ignoring it or move it"
|
|
685
699
|
)
|
|
686
700
|
tree = git(root, "write-tree", env=env)
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
)
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
701
|
+
changes = [
|
|
702
|
+
line.split("\t", 1)
|
|
703
|
+
for line in git(
|
|
704
|
+
root, "diff", "--name-status", "--no-renames", head, tree
|
|
705
|
+
).splitlines()
|
|
706
|
+
]
|
|
707
|
+
new_files = sorted(path for status, path in changes if status == "A")
|
|
708
|
+
changed_files = sorted(path for status, path in changes if status != "A")
|
|
709
|
+
sizes = {path: (root / path).lstat().st_size for path in new_files}
|
|
710
|
+
size = sum(sizes.values())
|
|
711
|
+
if size > NEW_FILES_LIMIT:
|
|
712
|
+
largest = sorted(new_files, key=lambda path: -sizes[path])
|
|
713
|
+
raise ValueError(
|
|
714
|
+
f"New files add {size // (1024 * 1024)} MiB to the snapshot, more than {NEW_FILES_LIMIT // (1024 * 1024)} MiB; add large outputs to .gitignore, such as "
|
|
715
|
+
+ ", ".join(largest[:5])
|
|
696
716
|
)
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
717
|
+
summary = {
|
|
718
|
+
"baseSha": head,
|
|
719
|
+
"changedFiles": changed_files,
|
|
720
|
+
"newFiles": new_files,
|
|
721
|
+
}
|
|
722
|
+
if preview:
|
|
723
|
+
return summary
|
|
703
724
|
sha = git(
|
|
704
725
|
root,
|
|
705
726
|
"commit-tree",
|
|
706
727
|
tree,
|
|
707
728
|
"-p",
|
|
708
729
|
head,
|
|
709
|
-
input=f"Bitfab replay {
|
|
730
|
+
input=f"Bitfab replay {request['id']}\n\n{json.dumps(request)}\n",
|
|
710
731
|
)
|
|
711
|
-
return {
|
|
732
|
+
return {**summary, "sha": sha}
|
|
712
733
|
|
|
713
734
|
|
|
714
735
|
def state_directory(root):
|
|
@@ -749,8 +770,7 @@ def find_run(record):
|
|
|
749
770
|
matches = [
|
|
750
771
|
run
|
|
751
772
|
for run in runs["workflow_runs"]
|
|
752
|
-
if run["head_sha"] == record["sha"]
|
|
753
|
-
and run.get("display_title") == f"Bitfab replay {record['id']}"
|
|
773
|
+
if run["head_sha"] == record["sha"] and run["head_branch"] == record["branch"]
|
|
754
774
|
]
|
|
755
775
|
if len(matches) > 1:
|
|
756
776
|
raise ValueError(
|
|
@@ -772,77 +792,115 @@ def status(record, *, fetch_result=True):
|
|
|
772
792
|
return record
|
|
773
793
|
if run["head_sha"] != record["sha"] or run["head_branch"] != record["branch"]:
|
|
774
794
|
raise ValueError("GitHub run does not match the recorded execution")
|
|
775
|
-
if run.get("display_title") != f"Bitfab replay {record['id']}":
|
|
776
|
-
if run["status"] == "completed":
|
|
777
|
-
raise ValueError("GitHub run does not match the recorded execution")
|
|
778
|
-
record.update(runId=run["id"], url=run["html_url"], state=run["status"])
|
|
779
|
-
return record
|
|
780
795
|
record.update(
|
|
781
796
|
runId=run["id"],
|
|
782
797
|
url=run["html_url"],
|
|
783
798
|
state=run["status"],
|
|
784
799
|
conclusion=run.get("conclusion"),
|
|
785
800
|
)
|
|
786
|
-
if
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
801
|
+
if fetch_result and record["state"] == "completed" and "exitCode" not in record:
|
|
802
|
+
outcome(record)
|
|
803
|
+
return record
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def replay_job(record):
|
|
807
|
+
jobs = api(
|
|
808
|
+
record["repository"], f"actions/runs/{record['runId']}/jobs?per_page=100"
|
|
809
|
+
)
|
|
810
|
+
jobs = (jobs or {}).get("jobs", [])
|
|
811
|
+
named = [job for job in jobs if job.get("name") == "replay"]
|
|
812
|
+
return (named or jobs or [None])[0]
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def job_log(record):
|
|
816
|
+
job = replay_job(record)
|
|
817
|
+
if job is None:
|
|
818
|
+
return []
|
|
819
|
+
for attempt in range(6):
|
|
794
820
|
try:
|
|
795
|
-
|
|
821
|
+
text = api(record["repository"], f"actions/jobs/{job['id']}/logs", raw=True)
|
|
822
|
+
return [LOG_TIMESTAMP.sub("", line) for line in text.splitlines()]
|
|
796
823
|
except RuntimeError:
|
|
797
|
-
if
|
|
824
|
+
if attempt == 5:
|
|
798
825
|
raise
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
if success:
|
|
802
|
-
raise ValueError("The GitHub job succeeded but left no replay result")
|
|
803
|
-
record["resultChecked"] = True
|
|
804
|
-
return record
|
|
805
|
-
if (
|
|
806
|
-
result.get("executionId") != record["id"]
|
|
807
|
-
or result.get("commitSha") != record["sha"]
|
|
808
|
-
):
|
|
809
|
-
raise ValueError("Replay result does not match this execution")
|
|
810
|
-
if record["request"].get("check"):
|
|
811
|
-
if result.get("check") != "passed" or not isinstance(
|
|
812
|
-
result.get("resolved"), int
|
|
813
|
-
):
|
|
814
|
-
raise ValueError("Cloud check result does not match this execution")
|
|
815
|
-
record["check"] = "passed"
|
|
816
|
-
record["resolved"] = result["resolved"]
|
|
817
|
-
return record
|
|
818
|
-
if not UUID.fullmatch(result.get("testRunId", "")):
|
|
819
|
-
raise ValueError("Replay result does not match this execution")
|
|
820
|
-
counts = replay_counts(result)
|
|
821
|
-
record["testRunId"] = result["testRunId"]
|
|
822
|
-
if isinstance(result.get("stoppedEarly"), str):
|
|
823
|
-
record["stoppedEarly"] = result["stoppedEarly"]
|
|
824
|
-
if counts is not None:
|
|
825
|
-
record["replayed"], record["errored"] = counts
|
|
826
|
-
return record
|
|
826
|
+
time.sleep(POLL_SECONDS)
|
|
827
|
+
return []
|
|
827
828
|
|
|
828
829
|
|
|
829
|
-
def read_result(
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
830
|
+
def read_result(lines):
|
|
831
|
+
chunks = [
|
|
832
|
+
line[len(RESULT_LINE) :] for line in lines if line.startswith(RESULT_LINE)
|
|
833
|
+
]
|
|
834
|
+
if not chunks:
|
|
835
|
+
return None
|
|
836
|
+
try:
|
|
837
|
+
return json.loads(
|
|
838
|
+
zlib.decompress(base64.b64decode("".join(chunks), validate=True))
|
|
839
|
+
)
|
|
840
|
+
except ValueError as error:
|
|
841
|
+
raise ValueError("The runner left an unreadable replay result") from error
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def replay_output(lines):
|
|
845
|
+
if OUTPUT_BEGIN not in lines:
|
|
846
|
+
return None
|
|
847
|
+
start = lines.index(OUTPUT_BEGIN) + 1
|
|
848
|
+
end = lines.index(OUTPUT_END, start) if OUTPUT_END in lines[start:] else len(lines)
|
|
849
|
+
return lines[start:end]
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def outcome(record):
|
|
853
|
+
lines = job_log(record)
|
|
854
|
+
result = read_result(lines)
|
|
855
|
+
if result is None:
|
|
856
|
+
record["exitCode"] = 1
|
|
857
|
+
return {"record": record, "lines": lines, "result": None}
|
|
858
|
+
if (
|
|
859
|
+
result.get("executionId") != record["id"]
|
|
860
|
+
or result.get("commitSha") != record["sha"]
|
|
861
|
+
):
|
|
862
|
+
raise ValueError("Replay result does not match this execution")
|
|
863
|
+
record["exitCode"] = result["exitCode"]
|
|
864
|
+
parsed = last_json(result.get("stdout", "")) or {}
|
|
865
|
+
test_run = result.get("testRunId") or parsed.get(
|
|
866
|
+
"testRunId", parsed.get("test_run_id")
|
|
867
|
+
)
|
|
868
|
+
if isinstance(test_run, str) and UUID.fullmatch(test_run):
|
|
869
|
+
record["testRunId"] = test_run
|
|
870
|
+
if result.get("stoppedEarly"):
|
|
871
|
+
record["stoppedEarly"] = result["stoppedEarly"]
|
|
872
|
+
return {"record": record, "lines": lines, "result": result}
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def print_outcome(found):
|
|
876
|
+
record, lines, result = found["record"], found["lines"], found["result"]
|
|
877
|
+
if result is None:
|
|
878
|
+
errors = [
|
|
879
|
+
index for index, line in enumerate(lines) if line.startswith("##[error]")
|
|
880
|
+
]
|
|
881
|
+
shown = lines[: errors[0] + 1] if errors else lines
|
|
882
|
+
tail = [line for line in shown if line.strip()][-LOG_TAIL_LINES:]
|
|
883
|
+
for line in tail:
|
|
884
|
+
print(line, file=sys.stderr)
|
|
885
|
+
progress(
|
|
886
|
+
f"The run stopped before the replay finished ({record.get('conclusion')}); the end of its log is above. Full log: {record.get('url')}"
|
|
887
|
+
)
|
|
888
|
+
return 1
|
|
889
|
+
for line in replay_output(lines) or []:
|
|
890
|
+
print(line, file=sys.stderr)
|
|
891
|
+
sys.stderr.flush()
|
|
892
|
+
print(result.get("stdout", ""), end="", flush=True)
|
|
893
|
+
if record.get("stoppedEarly"):
|
|
894
|
+
progress(
|
|
895
|
+
f"Replay stopped early: {record['stoppedEarly']}. Traces that finished are saved"
|
|
896
|
+
+ (
|
|
897
|
+
f" in experiment {record['testRunId']}"
|
|
898
|
+
if record.get("testRunId")
|
|
899
|
+
else ""
|
|
900
|
+
)
|
|
901
|
+
)
|
|
902
|
+
return result["exitCode"] or 1
|
|
903
|
+
return result["exitCode"]
|
|
846
904
|
|
|
847
905
|
|
|
848
906
|
def cleanup(root, record):
|
|
@@ -901,6 +959,123 @@ def preflight(repo, workflow):
|
|
|
901
959
|
)
|
|
902
960
|
|
|
903
961
|
|
|
962
|
+
def describe_snapshot(source):
|
|
963
|
+
changed, new = len(source["changedFiles"]), len(source["newFiles"])
|
|
964
|
+
progress(
|
|
965
|
+
f"Snapshot of {source['baseSha'][:12]} with {changed} changed and {new} new files"
|
|
966
|
+
+ (
|
|
967
|
+
f" ({', '.join(source['newFiles'][:5])}{', ...' if new > 5 else ''})"
|
|
968
|
+
if new
|
|
969
|
+
else ""
|
|
970
|
+
)
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def submit(root, repo, workflow, parsed, path):
|
|
975
|
+
execution_id = parsed["id"]
|
|
976
|
+
request, files = replay_request(root, parsed)
|
|
977
|
+
if "--cloud-preview" in parsed["switches"]:
|
|
978
|
+
return {
|
|
979
|
+
"preview": True,
|
|
980
|
+
"repository": repo,
|
|
981
|
+
**snapshot(root, workflow, files, request, preview=True),
|
|
982
|
+
}
|
|
983
|
+
if path.exists():
|
|
984
|
+
record = json.loads(path.read_text())
|
|
985
|
+
if record["request"] != request or record["repository"].lower() != repo.lower():
|
|
986
|
+
raise ValueError("Execution ID already belongs to a different request")
|
|
987
|
+
status(record, fetch_result=False)
|
|
988
|
+
if record["state"] == "prepared":
|
|
989
|
+
raise ValueError(
|
|
990
|
+
"Submission stopped before dispatch. Use --cloud-cleanup, then submit a new execution UUID"
|
|
991
|
+
)
|
|
992
|
+
return record
|
|
993
|
+
preflight(repo, workflow)
|
|
994
|
+
if dry_run(request["args"]):
|
|
995
|
+
progress(
|
|
996
|
+
"--dry-run dispatches a GitHub run that checks secrets and resolves the traces without replaying; it receives the workflow's secrets"
|
|
997
|
+
)
|
|
998
|
+
source = snapshot(root, workflow, files, request)
|
|
999
|
+
describe_snapshot(source)
|
|
1000
|
+
record = {
|
|
1001
|
+
"id": execution_id,
|
|
1002
|
+
"repository": repo,
|
|
1003
|
+
"workflow": workflow,
|
|
1004
|
+
"branch": PREFIX + execution_id,
|
|
1005
|
+
"request": request,
|
|
1006
|
+
"state": "prepared",
|
|
1007
|
+
**source,
|
|
1008
|
+
}
|
|
1009
|
+
save(path, record)
|
|
1010
|
+
progress(f"Execution {execution_id}; resume with --cloud-watch {execution_id}")
|
|
1011
|
+
ref = "refs/heads/" + record["branch"]
|
|
1012
|
+
try:
|
|
1013
|
+
git(
|
|
1014
|
+
root,
|
|
1015
|
+
"push",
|
|
1016
|
+
f"--force-with-lease={ref}:",
|
|
1017
|
+
"origin",
|
|
1018
|
+
f"{record['sha']}:{ref}",
|
|
1019
|
+
)
|
|
1020
|
+
except RuntimeError as error:
|
|
1021
|
+
raise ValueError(
|
|
1022
|
+
f"Pushing the snapshot branch {record['branch']} to origin failed; check that git can push to {repo}"
|
|
1023
|
+
) from error
|
|
1024
|
+
record["state"] = "dispatch_unknown"
|
|
1025
|
+
save(path, record)
|
|
1026
|
+
payload = {"ref": record["branch"]}
|
|
1027
|
+
if DECLARED_INPUTS.search(
|
|
1028
|
+
within(root, f"{WORKFLOW_DIRECTORY}/{workflow}").read_text()
|
|
1029
|
+
):
|
|
1030
|
+
payload["inputs"] = {"execution_id": execution_id, "request": "-"}
|
|
1031
|
+
try:
|
|
1032
|
+
response = api(
|
|
1033
|
+
repo,
|
|
1034
|
+
f"actions/workflows/{workflow}/dispatches",
|
|
1035
|
+
method="POST",
|
|
1036
|
+
payload=payload,
|
|
1037
|
+
)
|
|
1038
|
+
except RuntimeError as error:
|
|
1039
|
+
raise ValueError(
|
|
1040
|
+
f"Dispatching {workflow} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-watch {execution_id}"
|
|
1041
|
+
) from error
|
|
1042
|
+
if isinstance(response, dict) and response.get("workflow_run_id"):
|
|
1043
|
+
record["runId"] = response["workflow_run_id"]
|
|
1044
|
+
record["url"] = response.get("html_url")
|
|
1045
|
+
record["state"] = "queued"
|
|
1046
|
+
return record
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
def report_steps(record, shown):
|
|
1050
|
+
job = replay_job(record)
|
|
1051
|
+
for step in (job or {}).get("steps", []):
|
|
1052
|
+
if step.get("status") != "queued" and step.get("name") not in shown:
|
|
1053
|
+
shown.add(step.get("name"))
|
|
1054
|
+
progress(step.get("name"))
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def wait(record, path):
|
|
1058
|
+
started = time.monotonic()
|
|
1059
|
+
announced, steps = None, set()
|
|
1060
|
+
while record["state"] != "completed":
|
|
1061
|
+
status(record, fetch_result=False)
|
|
1062
|
+
save(path, record)
|
|
1063
|
+
if record["state"] != announced and record.get("url"):
|
|
1064
|
+
announced = record["state"]
|
|
1065
|
+
progress(f"{announced.replace('_', ' ').capitalize()}: {record['url']}")
|
|
1066
|
+
if record["state"] == "in_progress":
|
|
1067
|
+
report_steps(record, steps)
|
|
1068
|
+
if (
|
|
1069
|
+
record["state"] == "dispatch_unknown"
|
|
1070
|
+
and time.monotonic() - started >= 10 * 60
|
|
1071
|
+
):
|
|
1072
|
+
raise ValueError(
|
|
1073
|
+
f"GitHub has shown no run for this dispatch after 10 minutes. Check Actions, then resume with --cloud-watch {record['id']} or remove the snapshot branch with --cloud-cleanup {record['id']}"
|
|
1074
|
+
)
|
|
1075
|
+
if record["state"] != "completed":
|
|
1076
|
+
time.sleep(POLL_SECONDS)
|
|
1077
|
+
|
|
1078
|
+
|
|
904
1079
|
def run_cli(argv):
|
|
905
1080
|
parsed = parse(argv)
|
|
906
1081
|
operation, execution_id = parsed["operation"], parsed["id"]
|
|
@@ -913,99 +1088,9 @@ def run_cli(argv):
|
|
|
913
1088
|
path = directory / f"{execution_id}.json"
|
|
914
1089
|
with execution_lock(directory, execution_id):
|
|
915
1090
|
if operation == "submit":
|
|
916
|
-
|
|
917
|
-
if "
|
|
918
|
-
return
|
|
919
|
-
"dryRun": True,
|
|
920
|
-
"repository": repo,
|
|
921
|
-
**snapshot(
|
|
922
|
-
root,
|
|
923
|
-
workflow,
|
|
924
|
-
parsed["includes"],
|
|
925
|
-
files,
|
|
926
|
-
execution_id,
|
|
927
|
-
dry_run=True,
|
|
928
|
-
),
|
|
929
|
-
}
|
|
930
|
-
if path.exists():
|
|
931
|
-
record = json.loads(path.read_text())
|
|
932
|
-
if (
|
|
933
|
-
record["request"] != request
|
|
934
|
-
or record["repository"].lower() != repo.lower()
|
|
935
|
-
):
|
|
936
|
-
raise ValueError(
|
|
937
|
-
"Execution ID already belongs to a different request"
|
|
938
|
-
)
|
|
939
|
-
status(record)
|
|
940
|
-
if record["state"] == "prepared":
|
|
941
|
-
raise ValueError(
|
|
942
|
-
"Submission stopped before dispatch. Use --cloud-cleanup, then submit a new execution UUID"
|
|
943
|
-
)
|
|
944
|
-
else:
|
|
945
|
-
preflight(repo, workflow)
|
|
946
|
-
if request.get("check"):
|
|
947
|
-
print(
|
|
948
|
-
"Cloud dry run: dispatching a GitHub run that checks secrets and resolves the traces without replaying. It receives the workflow's secrets. Use --cloud-dry-run to only review the snapshot.",
|
|
949
|
-
file=sys.stderr,
|
|
950
|
-
flush=True,
|
|
951
|
-
)
|
|
952
|
-
source = snapshot(
|
|
953
|
-
root, workflow, parsed["includes"], files, execution_id
|
|
954
|
-
)
|
|
955
|
-
record = {
|
|
956
|
-
"id": execution_id,
|
|
957
|
-
"repository": repo,
|
|
958
|
-
"workflow": workflow,
|
|
959
|
-
"branch": PREFIX + execution_id,
|
|
960
|
-
"request": request,
|
|
961
|
-
"state": "prepared",
|
|
962
|
-
**source,
|
|
963
|
-
}
|
|
964
|
-
if "--fail-on-error" in parsed["switches"]:
|
|
965
|
-
record["failOnError"] = True
|
|
966
|
-
save(path, record)
|
|
967
|
-
print(
|
|
968
|
-
f"Cloud execution {execution_id}. Recover with --cloud-status {execution_id}",
|
|
969
|
-
file=sys.stderr,
|
|
970
|
-
flush=True,
|
|
971
|
-
)
|
|
972
|
-
ref = "refs/heads/" + record["branch"]
|
|
973
|
-
try:
|
|
974
|
-
git(
|
|
975
|
-
root,
|
|
976
|
-
"push",
|
|
977
|
-
f"--force-with-lease={ref}:",
|
|
978
|
-
"origin",
|
|
979
|
-
f"{record['sha']}:{ref}",
|
|
980
|
-
)
|
|
981
|
-
except RuntimeError as error:
|
|
982
|
-
raise ValueError(
|
|
983
|
-
f"Pushing the snapshot branch {record['branch']} to origin failed; check that git can push to {repo}"
|
|
984
|
-
) from error
|
|
985
|
-
record["state"] = "dispatch_unknown"
|
|
986
|
-
save(path, record)
|
|
987
|
-
encoded = base64.b64encode(json.dumps(request).encode()).decode()
|
|
988
|
-
try:
|
|
989
|
-
response = api(
|
|
990
|
-
repo,
|
|
991
|
-
f"actions/workflows/{workflow}/dispatches",
|
|
992
|
-
method="POST",
|
|
993
|
-
payload={
|
|
994
|
-
"ref": record["branch"],
|
|
995
|
-
"inputs": {
|
|
996
|
-
"execution_id": execution_id,
|
|
997
|
-
"request": encoded,
|
|
998
|
-
},
|
|
999
|
-
},
|
|
1000
|
-
)
|
|
1001
|
-
except RuntimeError as error:
|
|
1002
|
-
raise ValueError(
|
|
1003
|
-
f"Dispatching {workflow} on {record['branch']} failed{http_status(error)}; the GitHub account needs write access to Actions, and the registered workflow must accept workflow_dispatch. Check the Actions tab, then resume with --cloud-status {execution_id}"
|
|
1004
|
-
) from error
|
|
1005
|
-
if isinstance(response, dict) and response.get("workflow_run_id"):
|
|
1006
|
-
record["runId"] = response["workflow_run_id"]
|
|
1007
|
-
record["url"] = response.get("html_url")
|
|
1008
|
-
record["state"] = "queued"
|
|
1091
|
+
record = submit(root, repo, workflow, parsed, path)
|
|
1092
|
+
if record.get("preview"):
|
|
1093
|
+
return record, None
|
|
1009
1094
|
else:
|
|
1010
1095
|
if not path.exists():
|
|
1011
1096
|
raise ValueError(
|
|
@@ -1014,7 +1099,7 @@ def run_cli(argv):
|
|
|
1014
1099
|
record = json.loads(path.read_text())
|
|
1015
1100
|
if record["repository"].lower() != repo.lower():
|
|
1016
1101
|
raise ValueError("origin no longer matches the execution repository")
|
|
1017
|
-
status(record, fetch_result=operation
|
|
1102
|
+
status(record, fetch_result=operation == "status")
|
|
1018
1103
|
if operation == "cancel" and record["state"] != "completed":
|
|
1019
1104
|
if not record.get("runId"):
|
|
1020
1105
|
raise ValueError(
|
|
@@ -1023,27 +1108,13 @@ def run_cli(argv):
|
|
|
1023
1108
|
api(repo, f"actions/runs/{record['runId']}/cancel", method="POST")
|
|
1024
1109
|
record["state"] = "cancel_requested"
|
|
1025
1110
|
save(path, record)
|
|
1026
|
-
|
|
1027
|
-
return record
|
|
1028
|
-
if operation == "watch" or (
|
|
1111
|
+
attached = operation == "watch" or (
|
|
1029
1112
|
operation == "submit" and "--cloud-detach" not in parsed["switches"]
|
|
1030
|
-
)
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
if (
|
|
1036
|
-
record["state"] == "dispatch_unknown"
|
|
1037
|
-
and time.monotonic() - started >= 10 * 60
|
|
1038
|
-
):
|
|
1039
|
-
raise ValueError(
|
|
1040
|
-
"GitHub has shown no run for this dispatch after 10 minutes. Check Actions, then resume with --cloud-watch "
|
|
1041
|
-
+ execution_id
|
|
1042
|
-
+ " or remove the snapshot branch with --cloud-cleanup "
|
|
1043
|
-
+ execution_id
|
|
1044
|
-
)
|
|
1045
|
-
if record["state"] != "completed":
|
|
1046
|
-
time.sleep(5)
|
|
1113
|
+
)
|
|
1114
|
+
found = None
|
|
1115
|
+
if attached and record["state"] != "prepared":
|
|
1116
|
+
wait(record, path)
|
|
1117
|
+
found = outcome(record)
|
|
1047
1118
|
if record["state"] == "completed" or (
|
|
1048
1119
|
operation == "cleanup" and record["state"] == "prepared"
|
|
1049
1120
|
):
|
|
@@ -1051,7 +1122,7 @@ def run_cli(argv):
|
|
|
1051
1122
|
elif operation == "cleanup":
|
|
1052
1123
|
raise ValueError("Execution is not completed; no branch was deleted")
|
|
1053
1124
|
save(path, record)
|
|
1054
|
-
return record
|
|
1125
|
+
return record, found
|
|
1055
1126
|
|
|
1056
1127
|
|
|
1057
1128
|
def replay_command():
|
|
@@ -1065,25 +1136,29 @@ def replay_command():
|
|
|
1065
1136
|
or not all(isinstance(part, str) and part for part in value)
|
|
1066
1137
|
):
|
|
1067
1138
|
raise ValueError(
|
|
1068
|
-
"Start the
|
|
1139
|
+
"Start the replay step with the SDK's bitfab-replay command and --cloud-execute, so the runner knows how to start the replay"
|
|
1069
1140
|
)
|
|
1070
1141
|
return value
|
|
1071
1142
|
|
|
1072
1143
|
|
|
1073
|
-
def
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
if
|
|
1144
|
+
def snapshot_request(root, commit):
|
|
1145
|
+
message = git(root, "log", "-1", "--format=%B", commit)
|
|
1146
|
+
_, _, body = message.partition("\n\n")
|
|
1147
|
+
try:
|
|
1148
|
+
request = json.loads(body)
|
|
1149
|
+
except ValueError:
|
|
1150
|
+
request = None
|
|
1151
|
+
if not isinstance(request, dict) or request.get("version") != REQUEST_VERSION:
|
|
1152
|
+
version = request.get("version", 0) if isinstance(request, dict) else 0
|
|
1153
|
+
newer = isinstance(version, int) and version > REQUEST_VERSION
|
|
1081
1154
|
raise ValueError(
|
|
1082
|
-
"This replay was submitted by
|
|
1155
|
+
"This replay was submitted by a "
|
|
1156
|
+
+ ("newer" if newer else "different")
|
|
1157
|
+
+ " SDK than the one the snapshot installs; match the SDK you run locally to the one in the lockfile"
|
|
1083
1158
|
)
|
|
1084
|
-
if
|
|
1159
|
+
if os.environ.get("GITHUB_REF_NAME") != PREFIX + str(request.get("id")):
|
|
1085
1160
|
raise ValueError(
|
|
1086
|
-
"
|
|
1161
|
+
"The runner replays only the snapshot branch its request names; start replays with bitfab-replay --cloud"
|
|
1087
1162
|
)
|
|
1088
1163
|
validate_arguments(request["args"])
|
|
1089
1164
|
return request
|
|
@@ -1092,45 +1167,43 @@ def dispatched_request():
|
|
|
1092
1167
|
def execute():
|
|
1093
1168
|
if os.environ.get("GITHUB_RUN_ATTEMPT") != "1":
|
|
1094
1169
|
raise ValueError("Submit a new replay instead of rerunning an Actions job")
|
|
1095
|
-
request = dispatched_request()
|
|
1096
1170
|
root = root_directory()
|
|
1097
1171
|
commit = os.environ["GITHUB_SHA"]
|
|
1098
1172
|
if git(root, "rev-parse", "HEAD") != commit:
|
|
1099
1173
|
raise ValueError("Runner checkout does not match the dispatched snapshot SHA")
|
|
1174
|
+
request = snapshot_request(root, commit)
|
|
1100
1175
|
directory = within(root, request["cwd"])
|
|
1101
1176
|
check_secrets(root)
|
|
1102
|
-
names = {value
|
|
1177
|
+
names = {option_name(value) for value in request["args"]}
|
|
1103
1178
|
args = [
|
|
1104
1179
|
*replay_command(),
|
|
1105
1180
|
*request["args"],
|
|
1106
1181
|
*([] if names & set(CODE_CHANGE_FLAGS) else ["--no-code-change"]),
|
|
1107
1182
|
]
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
summary = {**identity, **run_check(directory, args)}
|
|
1111
|
-
write_result(summary)
|
|
1112
|
-
return summary
|
|
1183
|
+
if dry_run(request["args"]):
|
|
1184
|
+
run_check_command(directory)
|
|
1113
1185
|
experiment = {}
|
|
1114
1186
|
previous = signal.signal(signal.SIGTERM, raise_interrupt)
|
|
1187
|
+
print(OUTPUT_BEGIN, flush=True)
|
|
1115
1188
|
try:
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
}
|
|
1120
|
-
except BaseException as error:
|
|
1121
|
-
if UUID.fullmatch(experiment.get("id", "")):
|
|
1122
|
-
write_result(
|
|
1123
|
-
{
|
|
1124
|
-
**identity,
|
|
1125
|
-
"testRunId": experiment["id"],
|
|
1126
|
-
"stoppedEarly": (str(error) or type(error).__name__)[:300],
|
|
1127
|
-
}
|
|
1128
|
-
)
|
|
1129
|
-
raise
|
|
1189
|
+
code, stdout, stopped = run_command(
|
|
1190
|
+
directory, args, request.get("timeoutMinutes"), experiment
|
|
1191
|
+
)
|
|
1130
1192
|
finally:
|
|
1193
|
+
print(OUTPUT_END, flush=True)
|
|
1131
1194
|
signal.signal(signal.SIGTERM, previous)
|
|
1132
|
-
|
|
1133
|
-
|
|
1195
|
+
result = {
|
|
1196
|
+
"executionId": request["id"],
|
|
1197
|
+
"commitSha": commit,
|
|
1198
|
+
"exitCode": code,
|
|
1199
|
+
"stdout": stdout,
|
|
1200
|
+
}
|
|
1201
|
+
if stopped:
|
|
1202
|
+
result["stoppedEarly"] = stopped
|
|
1203
|
+
if UUID.fullmatch(experiment.get("id", "")):
|
|
1204
|
+
result["testRunId"] = experiment["id"]
|
|
1205
|
+
write_result(result)
|
|
1206
|
+
return result
|
|
1134
1207
|
|
|
1135
1208
|
|
|
1136
1209
|
def running_workflow(root):
|
|
@@ -1153,26 +1226,16 @@ def check_secrets(root):
|
|
|
1153
1226
|
)
|
|
1154
1227
|
|
|
1155
1228
|
|
|
1156
|
-
def
|
|
1229
|
+
def run_check_command(directory):
|
|
1157
1230
|
check = os.environ.get(CHECK_COMMAND_ENV, "").strip()
|
|
1158
|
-
if check:
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
).
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
items = result.get("items")
|
|
1167
|
-
if not isinstance(items, list):
|
|
1168
|
-
raise ValueError("The replay dry run did not return its resolved items")
|
|
1169
|
-
errors = item_errors(items)
|
|
1170
|
-
report_item_errors(errors)
|
|
1171
|
-
if errors:
|
|
1172
|
-
raise ValueError(
|
|
1173
|
-
f"{len(errors)} of {len(items)} traces failed to resolve; the errors are above"
|
|
1174
|
-
)
|
|
1175
|
-
return {"check": "passed", "resolved": len(items)}
|
|
1231
|
+
if not check:
|
|
1232
|
+
return
|
|
1233
|
+
print(f"Running {CHECK_COMMAND_ENV}: {check}", flush=True)
|
|
1234
|
+
code = subprocess.run(
|
|
1235
|
+
shlex.split(check), cwd=directory, stdin=subprocess.DEVNULL, check=False
|
|
1236
|
+
).returncode
|
|
1237
|
+
if code:
|
|
1238
|
+
raise ValueError(f"{CHECK_COMMAND_ENV} exited {code}; its output is above")
|
|
1176
1239
|
|
|
1177
1240
|
|
|
1178
1241
|
def item_errors(items):
|
|
@@ -1200,24 +1263,16 @@ def item_errors(items):
|
|
|
1200
1263
|
return errors
|
|
1201
1264
|
|
|
1202
1265
|
|
|
1203
|
-
def
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
sys.stderr.flush()
|
|
1214
|
-
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
1215
|
-
if summary:
|
|
1216
|
-
with Path(summary).open("a") as file:
|
|
1217
|
-
file.write(
|
|
1218
|
-
"\n#### Errored items\n\n"
|
|
1219
|
-
+ "".join(f"- `{line.replace('`', chr(39))}`\n" for line in lines)
|
|
1220
|
-
)
|
|
1266
|
+
def item_errored(item):
|
|
1267
|
+
return isinstance(item, dict) and any(
|
|
1268
|
+
item.get(field) is not None for field in ITEM_ERROR_FIELDS
|
|
1269
|
+
)
|
|
1270
|
+
|
|
1271
|
+
|
|
1272
|
+
def carried_over(item):
|
|
1273
|
+
return isinstance(item, dict) and (
|
|
1274
|
+
item.get("carriedOver") is True or item.get("carried_over") is True
|
|
1275
|
+
)
|
|
1221
1276
|
|
|
1222
1277
|
|
|
1223
1278
|
def raise_interrupt(signum, frame):
|
|
@@ -1226,67 +1281,69 @@ def raise_interrupt(signum, frame):
|
|
|
1226
1281
|
|
|
1227
1282
|
def forward_stderr(stream, experiment):
|
|
1228
1283
|
for line in iter(stream.readline, b""):
|
|
1229
|
-
sys.
|
|
1230
|
-
sys.
|
|
1284
|
+
sys.stdout.write(line.decode(errors="replace"))
|
|
1285
|
+
sys.stdout.flush()
|
|
1231
1286
|
if "id" not in experiment:
|
|
1232
1287
|
match = EXPERIMENT_LINE.match(line)
|
|
1233
1288
|
if match:
|
|
1234
1289
|
experiment["id"] = match[1].decode()
|
|
1235
1290
|
|
|
1236
1291
|
|
|
1237
|
-
def
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
if "
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
f"Commit: `{summary['commitSha']}`",
|
|
1250
|
-
f"Stopped early: {summary['stoppedEarly']}. Traces that finished are saved in this test run."
|
|
1251
|
-
if "stoppedEarly" in summary
|
|
1252
|
-
else f"Replayed: {summary['replayed']}, errored: {summary['errored']}",
|
|
1253
|
-
]
|
|
1254
|
-
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as file:
|
|
1255
|
-
file.write(f"### {heading}\n\n" + "\n\n".join(lines) + "\n")
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
def run_replay(directory, args, timeout, experiment):
|
|
1259
|
-
result = run_command(directory, args, timeout, experiment)
|
|
1260
|
-
test_run = result.get("testRunId", result.get("test_run_id"))
|
|
1261
|
-
if not isinstance(test_run, str) or not UUID.fullmatch(test_run):
|
|
1262
|
-
raise ValueError("Replay did not return a valid persisted test run UUID")
|
|
1263
|
-
items = result.get("items")
|
|
1264
|
-
if not isinstance(items, list):
|
|
1265
|
-
raise ValueError("Replay did not return its replayed items")
|
|
1266
|
-
replayed = [item for item in items if not carried_over(item)]
|
|
1267
|
-
errors = item_errors(replayed)
|
|
1268
|
-
report_item_errors(errors)
|
|
1269
|
-
return {"testRunId": test_run, "replayed": len(replayed), "errored": len(errors)}
|
|
1292
|
+
def last_json(text):
|
|
1293
|
+
decoder = json.JSONDecoder()
|
|
1294
|
+
result = None
|
|
1295
|
+
for offset in [0, *[i + 1 for i, value in enumerate(text) if value == "\n"]]:
|
|
1296
|
+
if text.startswith("{", offset):
|
|
1297
|
+
try:
|
|
1298
|
+
value, end = decoder.raw_decode(text, offset)
|
|
1299
|
+
if not text[end:].strip() and isinstance(value, dict):
|
|
1300
|
+
result = value
|
|
1301
|
+
except json.JSONDecodeError:
|
|
1302
|
+
pass
|
|
1303
|
+
return result
|
|
1270
1304
|
|
|
1271
1305
|
|
|
1272
|
-
def
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1306
|
+
def write_result(result):
|
|
1307
|
+
encoded = base64.b64encode(zlib.compress(json.dumps(result).encode())).decode()
|
|
1308
|
+
for start in range(0, len(encoded), RESULT_CHUNK):
|
|
1309
|
+
print(RESULT_LINE + encoded[start : start + RESULT_CHUNK])
|
|
1310
|
+
sys.stdout.flush()
|
|
1311
|
+
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
1312
|
+
if summary:
|
|
1313
|
+
with Path(summary).open("a") as file:
|
|
1314
|
+
file.write(summary_markdown(result))
|
|
1276
1315
|
|
|
1277
1316
|
|
|
1278
|
-
def
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1317
|
+
def summary_markdown(result):
|
|
1318
|
+
parsed = last_json(result["stdout"]) or {}
|
|
1319
|
+
test_run = result.get("testRunId") or parsed.get(
|
|
1320
|
+
"testRunId", parsed.get("test_run_id")
|
|
1321
|
+
)
|
|
1322
|
+
items = parsed.get("items") if isinstance(parsed.get("items"), list) else []
|
|
1323
|
+
replayed = [item for item in items if not carried_over(item)]
|
|
1324
|
+
errors = item_errors(replayed)
|
|
1325
|
+
lines = [f"Commit: `{result['commitSha']}`", f"Exit code: {result['exitCode']}"]
|
|
1326
|
+
if test_run:
|
|
1327
|
+
lines.insert(0, f"Test run: `{test_run}`")
|
|
1328
|
+
if result.get("stoppedEarly"):
|
|
1329
|
+
lines.append(f"Stopped early: {result['stoppedEarly']}")
|
|
1330
|
+
elif items:
|
|
1331
|
+
lines.append(f"Replayed: {len(replayed)}, errored: {len(errors)}")
|
|
1332
|
+
text = "### Bitfab replay\n\n" + "\n\n".join(lines) + "\n"
|
|
1333
|
+
if errors:
|
|
1334
|
+
shown = [
|
|
1335
|
+
f"trace {trace}: {error}" for trace, error in errors[:ITEM_ERROR_LINES]
|
|
1336
|
+
]
|
|
1337
|
+
if len(errors) > len(shown):
|
|
1338
|
+
shown.append(f"and {len(errors) - len(shown)} more errored items")
|
|
1339
|
+
text += "\n#### Errored items\n\n" + "".join(
|
|
1340
|
+
f"- `{line.replace('`', chr(39))}`\n" for line in shown
|
|
1286
1341
|
)
|
|
1342
|
+
return text
|
|
1287
1343
|
|
|
1288
1344
|
|
|
1289
1345
|
def run_command(directory, args, timeout, experiment):
|
|
1346
|
+
stopped = None
|
|
1290
1347
|
with tempfile.TemporaryFile() as output:
|
|
1291
1348
|
with subprocess.Popen(
|
|
1292
1349
|
args,
|
|
@@ -1303,84 +1360,37 @@ def run_command(directory, args, timeout, experiment):
|
|
|
1303
1360
|
try:
|
|
1304
1361
|
deadline = None if timeout is None else time.monotonic() + timeout * 60
|
|
1305
1362
|
while child.poll() is None:
|
|
1306
|
-
if os.fstat(output.fileno()).st_size >
|
|
1363
|
+
if os.fstat(output.fileno()).st_size > OUTPUT_LIMIT:
|
|
1307
1364
|
raise ValueError("Replay output exceeded 16 MiB")
|
|
1308
1365
|
if deadline is not None and time.monotonic() >= deadline:
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1366
|
+
stopped = f"it reached the {timeout}-minute --cloud-timeout"
|
|
1367
|
+
stop(child)
|
|
1368
|
+
break
|
|
1312
1369
|
time.sleep(0.1)
|
|
1313
|
-
|
|
1370
|
+
except KeyboardInterrupt:
|
|
1371
|
+
stopped = "the GitHub run was cancelled"
|
|
1372
|
+
stop(child)
|
|
1314
1373
|
except BaseException:
|
|
1315
|
-
|
|
1316
|
-
os.killpg(child.pid, signal.SIGTERM)
|
|
1317
|
-
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1318
|
-
child.wait(timeout=30)
|
|
1319
|
-
with contextlib.suppress(ProcessLookupError):
|
|
1320
|
-
os.killpg(child.pid, signal.SIGKILL)
|
|
1321
|
-
child.wait()
|
|
1374
|
+
stop(child)
|
|
1322
1375
|
raise
|
|
1323
1376
|
finally:
|
|
1324
1377
|
reader.join(timeout=5)
|
|
1325
|
-
|
|
1378
|
+
code = child.returncode
|
|
1379
|
+
if output.tell() > OUTPUT_LIMIT:
|
|
1326
1380
|
raise ValueError("Replay output exceeded 16 MiB")
|
|
1327
1381
|
output.seek(0)
|
|
1328
1382
|
text = output.read().decode(errors="replace")
|
|
1329
|
-
if code
|
|
1330
|
-
print_output_tail(text)
|
|
1331
|
-
raise ValueError(f"Replay command exited {code}; its output is above")
|
|
1332
|
-
decoder = json.JSONDecoder()
|
|
1333
|
-
result = None
|
|
1334
|
-
for offset in [0, *[i + 1 for i, value in enumerate(text) if value == "\n"]]:
|
|
1335
|
-
if text.startswith("{", offset):
|
|
1336
|
-
try:
|
|
1337
|
-
value, end = decoder.raw_decode(text, offset)
|
|
1338
|
-
if not text[end:].strip() and isinstance(value, dict):
|
|
1339
|
-
result = value
|
|
1340
|
-
except json.JSONDecodeError:
|
|
1341
|
-
pass
|
|
1342
|
-
if result is None:
|
|
1343
|
-
print_output_tail(text)
|
|
1344
|
-
raise ValueError("Replay did not print a JSON result; its output is above")
|
|
1345
|
-
return result
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
def item_errored(item):
|
|
1349
|
-
return isinstance(item, dict) and any(
|
|
1350
|
-
item.get(field) is not None for field in ITEM_ERROR_FIELDS
|
|
1351
|
-
)
|
|
1383
|
+
return (code if code >= 0 else 1), text, stopped
|
|
1352
1384
|
|
|
1353
1385
|
|
|
1354
|
-
def
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
)
|
|
1363
|
-
or errored > replayed
|
|
1364
|
-
):
|
|
1365
|
-
raise ValueError("Replay result has invalid item counts")
|
|
1366
|
-
return replayed, errored
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
def report_errored_items(record):
|
|
1370
|
-
replayed, errored = record.get("replayed"), record.get("errored")
|
|
1371
|
-
if not errored:
|
|
1372
|
-
return 0
|
|
1373
|
-
if errored == replayed:
|
|
1374
|
-
print(
|
|
1375
|
-
f"Cloud replay: every replayed trace errored ({errored} of {replayed}), so nothing ran successfully. Open test run {record.get('testRunId')} to see why.",
|
|
1376
|
-
file=sys.stderr,
|
|
1377
|
-
)
|
|
1378
|
-
return 1
|
|
1379
|
-
print(
|
|
1380
|
-
f"Cloud replay: {errored} of {replayed} replayed traces errored. Open test run {record.get('testRunId')} to see why.",
|
|
1381
|
-
file=sys.stderr,
|
|
1382
|
-
)
|
|
1383
|
-
return 0
|
|
1386
|
+
def stop(child):
|
|
1387
|
+
with contextlib.suppress(ProcessLookupError):
|
|
1388
|
+
os.killpg(child.pid, signal.SIGTERM)
|
|
1389
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1390
|
+
child.wait(timeout=30)
|
|
1391
|
+
with contextlib.suppress(ProcessLookupError):
|
|
1392
|
+
os.killpg(child.pid, signal.SIGKILL)
|
|
1393
|
+
child.wait()
|
|
1384
1394
|
|
|
1385
1395
|
|
|
1386
1396
|
PLAIN_SCALAR = re.compile(r"[A-Za-z0-9_./][A-Za-z0-9_ ./@*+=,()-]*")
|
|
@@ -1491,30 +1501,26 @@ def typescript_project(root, start):
|
|
|
1491
1501
|
"yarn": "yarn install --frozen-lockfile",
|
|
1492
1502
|
"npm": "npm ci" if lockfile else "npm install",
|
|
1493
1503
|
}[manager]
|
|
1494
|
-
node = {
|
|
1495
|
-
"name": "Set up Node.js",
|
|
1496
|
-
"uses": ACTIONS["node"],
|
|
1497
|
-
"with": {
|
|
1498
|
-
**version_input(root, start, "node", [".nvmrc", ".node-version"], "lts/*"),
|
|
1499
|
-
**(
|
|
1500
|
-
{"cache": manager}
|
|
1501
|
-
if manager in ("pnpm", "yarn", "npm") and lockfile
|
|
1502
|
-
else {}
|
|
1503
|
-
),
|
|
1504
|
-
},
|
|
1505
|
-
}
|
|
1506
1504
|
steps = []
|
|
1507
1505
|
if manager == "pnpm":
|
|
1508
|
-
steps.append({"
|
|
1506
|
+
steps.append({"uses": ACTIONS["pnpm"]})
|
|
1509
1507
|
if manager == "bun":
|
|
1510
|
-
steps.append({"
|
|
1508
|
+
steps.append({"uses": ACTIONS["bun"]})
|
|
1511
1509
|
steps += [
|
|
1512
|
-
node,
|
|
1513
1510
|
{
|
|
1514
|
-
"
|
|
1515
|
-
|
|
1516
|
-
|
|
1511
|
+
"uses": ACTIONS["node"],
|
|
1512
|
+
"with": {
|
|
1513
|
+
**version_input(
|
|
1514
|
+
root, start, "node", [".nvmrc", ".node-version"], "lts/*"
|
|
1515
|
+
),
|
|
1516
|
+
**(
|
|
1517
|
+
{"cache": manager}
|
|
1518
|
+
if manager in ("pnpm", "yarn", "npm") and lockfile
|
|
1519
|
+
else {}
|
|
1520
|
+
),
|
|
1521
|
+
},
|
|
1517
1522
|
},
|
|
1523
|
+
{**step_directory(root, directory or start), "run": install},
|
|
1518
1524
|
]
|
|
1519
1525
|
run = {
|
|
1520
1526
|
"pnpm": "pnpm exec bitfab-replay",
|
|
@@ -1531,20 +1537,18 @@ def python_project(root, start):
|
|
|
1531
1537
|
)
|
|
1532
1538
|
location = step_directory(root, directory or start)
|
|
1533
1539
|
python = {
|
|
1534
|
-
"name": "Set up Python",
|
|
1535
1540
|
"uses": ACTIONS["python"],
|
|
1536
1541
|
"with": version_input(root, start, "python", [".python-version"], "3.12"),
|
|
1537
1542
|
}
|
|
1538
1543
|
if lockfile == "uv.lock":
|
|
1539
1544
|
return [
|
|
1540
|
-
{"
|
|
1541
|
-
{
|
|
1545
|
+
{"uses": ACTIONS["uv"]},
|
|
1546
|
+
{**location, "run": "uv sync --frozen"},
|
|
1542
1547
|
], "uv run bitfab-replay"
|
|
1543
1548
|
if lockfile == "poetry.lock":
|
|
1544
1549
|
return [
|
|
1545
1550
|
python,
|
|
1546
1551
|
{
|
|
1547
|
-
"name": "Install dependencies",
|
|
1548
1552
|
**location,
|
|
1549
1553
|
"run": "pipx install poetry && poetry install --no-interaction",
|
|
1550
1554
|
},
|
|
@@ -1554,21 +1558,14 @@ def python_project(root, start):
|
|
|
1554
1558
|
if lockfile == "requirements.txt"
|
|
1555
1559
|
else "pip install ."
|
|
1556
1560
|
)
|
|
1557
|
-
return [
|
|
1558
|
-
python,
|
|
1559
|
-
{"name": "Install dependencies", **location, "run": install},
|
|
1560
|
-
], "bitfab-replay"
|
|
1561
|
+
return [python, {**location, "run": install}], "bitfab-replay"
|
|
1561
1562
|
|
|
1562
1563
|
|
|
1563
1564
|
def ruby_project(root, start):
|
|
1564
1565
|
directory, _ = find_upward(root, start, ["Gemfile"])
|
|
1565
1566
|
location = step_directory(root, directory or start)
|
|
1566
1567
|
return [
|
|
1567
|
-
{
|
|
1568
|
-
"name": "Set up Ruby",
|
|
1569
|
-
"uses": ACTIONS["ruby"],
|
|
1570
|
-
"with": {"bundler-cache": True, **location},
|
|
1571
|
-
}
|
|
1568
|
+
{"uses": ACTIONS["ruby"], "with": {"bundler-cache": True, **location}}
|
|
1572
1569
|
], "bundle exec bitfab-replay"
|
|
1573
1570
|
|
|
1574
1571
|
|
|
@@ -1579,7 +1576,30 @@ def go_project(root, start):
|
|
|
1579
1576
|
if name
|
|
1580
1577
|
else {"go-version": "stable"}
|
|
1581
1578
|
)
|
|
1582
|
-
return [{"
|
|
1579
|
+
return [{"uses": ACTIONS["go"], "with": version}], go_registry_command(root, start)
|
|
1580
|
+
|
|
1581
|
+
|
|
1582
|
+
def go_registry_command(root, start):
|
|
1583
|
+
programs = set()
|
|
1584
|
+
for directory, folders, names in os.walk(root):
|
|
1585
|
+
folders[:] = [
|
|
1586
|
+
folder
|
|
1587
|
+
for folder in folders
|
|
1588
|
+
if not folder.startswith(".")
|
|
1589
|
+
and folder not in ("node_modules", "vendor", "testdata")
|
|
1590
|
+
]
|
|
1591
|
+
for name in names:
|
|
1592
|
+
if not name.endswith(".go") or name.endswith("_test.go"):
|
|
1593
|
+
continue
|
|
1594
|
+
text = (Path(directory) / name).read_text(errors="replace")
|
|
1595
|
+
if "RunCloudReplayCLI(" in text and re.search(
|
|
1596
|
+
r"^package main\b", text, re.MULTILINE
|
|
1597
|
+
):
|
|
1598
|
+
programs.add(Path(directory))
|
|
1599
|
+
if len(programs) != 1:
|
|
1600
|
+
return None
|
|
1601
|
+
relative = os.path.relpath(programs.pop(), start)
|
|
1602
|
+
return f"go run {relative if relative.startswith('..') else './' + relative}"
|
|
1583
1603
|
|
|
1584
1604
|
|
|
1585
1605
|
PROJECTS = {
|
|
@@ -1608,47 +1628,22 @@ def push_triggers(root, workflow):
|
|
|
1608
1628
|
return found
|
|
1609
1629
|
|
|
1610
1630
|
|
|
1611
|
-
def
|
|
1612
|
-
return "${{ " + kind + "." + name + " }}"
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
def secret_env(names, prefix):
|
|
1616
|
-
return {name: reference("secrets", prefix + name) for name in names}
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
def replay_step(run, directory, env):
|
|
1620
|
-
return {
|
|
1621
|
-
"name": "Replay",
|
|
1622
|
-
**({} if directory == "." else {"working-directory": directory}),
|
|
1623
|
-
"run": f"{run} --cloud-execute",
|
|
1624
|
-
"env": env,
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
def workflow_document(steps, replay, *, runs_on="ubuntu-24.04", environment=None):
|
|
1631
|
+
def workflow_document(steps, run, directory, env, *, runs_on, environment):
|
|
1629
1632
|
job = {"runs-on": runs_on}
|
|
1630
1633
|
if environment:
|
|
1631
1634
|
job["environment"] = environment
|
|
1632
1635
|
job["steps"] = [
|
|
1636
|
+
{"uses": ACTIONS["checkout"], "with": {"persist-credentials": False}},
|
|
1637
|
+
*steps,
|
|
1633
1638
|
{
|
|
1634
|
-
|
|
1635
|
-
"
|
|
1636
|
-
"
|
|
1639
|
+
**({} if directory == "." else {"working-directory": directory}),
|
|
1640
|
+
"run": f"{run} --cloud-execute",
|
|
1641
|
+
"env": env,
|
|
1637
1642
|
},
|
|
1638
|
-
*steps,
|
|
1639
|
-
replay,
|
|
1640
1643
|
]
|
|
1641
1644
|
return {
|
|
1642
|
-
"name": "Bitfab
|
|
1643
|
-
"
|
|
1644
|
-
"on": {
|
|
1645
|
-
"workflow_dispatch": {
|
|
1646
|
-
"inputs": {
|
|
1647
|
-
"execution_id": {"required": True, "type": "string"},
|
|
1648
|
-
"request": {"required": True, "type": "string"},
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
},
|
|
1645
|
+
"name": "Bitfab replay",
|
|
1646
|
+
"on": "workflow_dispatch",
|
|
1652
1647
|
"permissions": {"contents": "read"},
|
|
1653
1648
|
"jobs": {"replay": job},
|
|
1654
1649
|
}
|
|
@@ -1657,7 +1652,7 @@ def workflow_document(steps, replay, *, runs_on="ubuntu-24.04", environment=None
|
|
|
1657
1652
|
def initialize(argv):
|
|
1658
1653
|
parser = argparse.ArgumentParser(
|
|
1659
1654
|
prog="bitfab-replay --cloud-init",
|
|
1660
|
-
description="Write .github/workflows/bitfab-replay.yml, the only file cloud replay keeps in the repository. It detects the runtime, package manager, and
|
|
1655
|
+
description="Write .github/workflows/bitfab-replay.yml, the only file cloud replay keeps in the repository, and list what is left to do. It detects the runtime, package manager, install command, and replay command from the directory you run it in. An existing workflow is left alone; edit it directly to change it.",
|
|
1661
1656
|
)
|
|
1662
1657
|
parser.add_argument(
|
|
1663
1658
|
"--secret",
|
|
@@ -1677,24 +1672,24 @@ def initialize(argv):
|
|
|
1677
1672
|
)
|
|
1678
1673
|
parser.add_argument(
|
|
1679
1674
|
"--run",
|
|
1680
|
-
help='Command that starts the SDK\'s bitfab-replay on the runner, such as "pnpm exec bitfab-replay"; detected for
|
|
1675
|
+
help='Command that starts the SDK\'s bitfab-replay on the runner, such as "pnpm exec bitfab-replay"; detected for every SDK, including the Go program that calls RunCloudReplayCLI',
|
|
1681
1676
|
)
|
|
1682
1677
|
parser.add_argument("--runs-on", default="ubuntu-24.04", help="Runner label")
|
|
1683
1678
|
parser.add_argument(
|
|
1684
1679
|
"--check",
|
|
1685
|
-
help='Command the runner also runs
|
|
1680
|
+
help='Command the runner also runs on a --cloud --dry-run, such as "node scripts/checkBucket.js"',
|
|
1686
1681
|
)
|
|
1687
1682
|
args = parser.parse_args(argv)
|
|
1688
1683
|
root = root_directory()
|
|
1689
|
-
|
|
1690
|
-
return migrate(root, args)
|
|
1684
|
+
refuse_old_setup(root)
|
|
1691
1685
|
existing = find_workflow(root)
|
|
1692
1686
|
if existing is not None:
|
|
1693
1687
|
return {
|
|
1694
1688
|
"files": [],
|
|
1695
|
-
"updated": False,
|
|
1696
1689
|
"workflow": f"{WORKFLOW_DIRECTORY}/{existing}",
|
|
1697
|
-
"next":
|
|
1690
|
+
"next": [
|
|
1691
|
+
"Already set up. Edit the workflow directly to change its install steps, secrets, runner, or Environment"
|
|
1692
|
+
],
|
|
1698
1693
|
}
|
|
1699
1694
|
prefix = args.secret_prefix
|
|
1700
1695
|
if prefix and not re.fullmatch(r"[A-Z][A-Z0-9_]*_", prefix):
|
|
@@ -1713,191 +1708,83 @@ def initialize(argv):
|
|
|
1713
1708
|
raise ValueError(
|
|
1714
1709
|
'Pass --run with the command that starts your registry program on the runner, such as --run "go run ./cmd/registry"'
|
|
1715
1710
|
)
|
|
1716
|
-
env =
|
|
1711
|
+
env = {name: "${{ secrets." + prefix + name + " }}" for name in names}
|
|
1717
1712
|
if args.check:
|
|
1718
1713
|
env[CHECK_COMMAND_ENV] = args.check
|
|
1719
|
-
workflow = workflow_document(
|
|
1720
|
-
steps,
|
|
1721
|
-
replay_step(run, directory, env),
|
|
1722
|
-
runs_on=args.runs_on,
|
|
1723
|
-
environment=args.environment,
|
|
1724
|
-
)
|
|
1725
1714
|
path = f"{WORKFLOW_DIRECTORY}/{WORKFLOW}"
|
|
1726
1715
|
target = within(root, path)
|
|
1727
1716
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1728
1717
|
with target.open("x") as file:
|
|
1729
|
-
file.write(
|
|
1718
|
+
file.write(
|
|
1719
|
+
to_yaml(
|
|
1720
|
+
workflow_document(
|
|
1721
|
+
steps,
|
|
1722
|
+
run,
|
|
1723
|
+
directory,
|
|
1724
|
+
env,
|
|
1725
|
+
runs_on=args.runs_on,
|
|
1726
|
+
environment=args.environment,
|
|
1727
|
+
)
|
|
1728
|
+
)
|
|
1729
|
+
)
|
|
1730
|
+
secrets = sorted(prefix + name for name in names)
|
|
1731
|
+
triggers = push_triggers(root, WORKFLOW)
|
|
1732
|
+
try:
|
|
1733
|
+
where = secrets_page(repository(root), args.environment)
|
|
1734
|
+
except (ValueError, RuntimeError):
|
|
1735
|
+
where = "the repository's Settings, Secrets and variables, Actions"
|
|
1730
1736
|
return {
|
|
1731
1737
|
"files": [path],
|
|
1732
1738
|
"detected": {
|
|
1733
1739
|
"language": language,
|
|
1734
1740
|
"install": [step.get("run") or step["uses"] for step in steps],
|
|
1735
|
-
"replay":
|
|
1741
|
+
"replay": f"{run} --cloud-execute",
|
|
1736
1742
|
},
|
|
1737
|
-
"secrets":
|
|
1738
|
-
"
|
|
1739
|
-
"
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
removed = []
|
|
1753
|
-
for old in (OLD_CONFIG, OLD_SCRIPT):
|
|
1754
|
-
if within(root, old).exists():
|
|
1755
|
-
within(root, old).unlink()
|
|
1756
|
-
removed.append(old)
|
|
1757
|
-
return removed
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
def migrate(root, args):
|
|
1761
|
-
config = json.loads(within(root, OLD_CONFIG).read_text())
|
|
1762
|
-
name = config.get("workflow", WORKFLOW)
|
|
1763
|
-
path = f"{WORKFLOW_DIRECTORY}/{name}"
|
|
1764
|
-
text = within(root, path).read_text() if within(root, path).is_file() else ""
|
|
1765
|
-
try:
|
|
1766
|
-
workflow = json.loads(text)
|
|
1767
|
-
job = workflow["jobs"]["replay"]
|
|
1768
|
-
steps = job["steps"]
|
|
1769
|
-
except (ValueError, KeyError, TypeError) as error:
|
|
1770
|
-
if "--cloud-execute" in text and not any(
|
|
1771
|
-
old in text for old in (OLD_SCRIPT, OLD_UPLOAD_STEP, *OLD_RUNNER_ENV)
|
|
1772
|
-
):
|
|
1773
|
-
if config.get("checkCommand") and CHECK_COMMAND_ENV not in text:
|
|
1774
|
-
raise ValueError(
|
|
1775
|
-
f"{OLD_CONFIG} has a checkCommand the workflow lacks. Add {CHECK_COMMAND_ENV}: {shlex.join(config['checkCommand'])} to the Replay step's env, then run --cloud-init again"
|
|
1776
|
-
) from error
|
|
1777
|
-
return {
|
|
1778
|
-
"files": [],
|
|
1779
|
-
"removed": remove_old_files(root),
|
|
1780
|
-
"updated": True,
|
|
1781
|
-
**dropped_command(config),
|
|
1782
|
-
"next": "The workflow already holds these settings, so only the old files were removed",
|
|
1783
|
-
}
|
|
1784
|
-
raise ValueError(
|
|
1785
|
-
f"{path} is not the workflow --cloud-init generated. Make its Replay step run the SDK's bitfab-replay command with --cloud-execute and read each secret in its env, then delete {OLD_CONFIG}"
|
|
1786
|
-
) from error
|
|
1787
|
-
replay = [
|
|
1788
|
-
index
|
|
1789
|
-
for index, step in enumerate(steps)
|
|
1790
|
-
if isinstance(step, dict) and step.get("name") == "Replay"
|
|
1791
|
-
]
|
|
1792
|
-
if len(replay) != 1:
|
|
1793
|
-
raise ValueError(f"{path} must contain exactly one step named Replay")
|
|
1794
|
-
step = steps[replay[0]]
|
|
1795
|
-
words = shlex.split(step.get("run") or "")
|
|
1796
|
-
if args.run:
|
|
1797
|
-
run = args.run
|
|
1798
|
-
elif words[-1:] == ["--cloud-execute"] and OLD_SCRIPT not in step.get("run", ""):
|
|
1799
|
-
run = shlex.join(words[:-1])
|
|
1800
|
-
elif config.get("cliCommand"):
|
|
1801
|
-
run = shlex.join(config["cliCommand"])
|
|
1802
|
-
else:
|
|
1803
|
-
raise ValueError(
|
|
1804
|
-
'The Replay step does not start the SDK\'s bitfab-replay command; pass --run with the command that does, such as --run "npx --no-install bitfab-replay"'
|
|
1805
|
-
)
|
|
1806
|
-
prefix = config.get("secretPrefix", "")
|
|
1807
|
-
env = {
|
|
1808
|
-
**secret_env(config.get("secrets", []), prefix),
|
|
1809
|
-
**{
|
|
1810
|
-
key: value
|
|
1811
|
-
for key, value in (step.get("env") or {}).items()
|
|
1812
|
-
if key not in OLD_RUNNER_ENV
|
|
1813
|
-
},
|
|
1814
|
-
}
|
|
1815
|
-
for variable, source in (config.get("env") or {}).items():
|
|
1816
|
-
kind, target = next(iter(source.items()))
|
|
1817
|
-
env.setdefault(
|
|
1818
|
-
variable, reference("secrets" if kind == "secret" else "vars", target)
|
|
1819
|
-
)
|
|
1820
|
-
if config.get("checkCommand"):
|
|
1821
|
-
env[CHECK_COMMAND_ENV] = shlex.join(config["checkCommand"])
|
|
1822
|
-
kept = {
|
|
1823
|
-
key: value
|
|
1824
|
-
for key, value in step.items()
|
|
1825
|
-
if key not in ("name", "uses", "with", "run", "working-directory", "env")
|
|
1826
|
-
}
|
|
1827
|
-
job["steps"] = [
|
|
1828
|
-
{**replay_step(run, config.get("workingDirectory", "."), env), **kept}
|
|
1829
|
-
if index == replay[0]
|
|
1830
|
-
else entry
|
|
1831
|
-
for index, entry in enumerate(steps)
|
|
1832
|
-
if not (
|
|
1833
|
-
isinstance(entry, dict)
|
|
1834
|
-
and entry.get("name") == OLD_UPLOAD_STEP
|
|
1835
|
-
and str(entry.get("uses", "")).startswith("actions/upload-artifact@")
|
|
1836
|
-
)
|
|
1837
|
-
]
|
|
1838
|
-
if job.get("timeout-minutes") == OLD_JOB_TIMEOUT:
|
|
1839
|
-
del job["timeout-minutes"]
|
|
1840
|
-
replace_file(within(root, path), to_yaml(workflow))
|
|
1841
|
-
result = {
|
|
1842
|
-
"files": [path],
|
|
1843
|
-
"removed": remove_old_files(root),
|
|
1844
|
-
"updated": True,
|
|
1845
|
-
"next": "Review the diff. Replays run the workflow in their own snapshot, so the change applies without merging",
|
|
1743
|
+
"secrets": secrets,
|
|
1744
|
+
"reviewPushTriggers": triggers,
|
|
1745
|
+
"next": [
|
|
1746
|
+
"Compare the install steps with your CI and add anything detection missed, such as a build or code-generation step",
|
|
1747
|
+
f"Create the secrets {', '.join(secrets)} at {where}, or copy them from a local file with bitfab-replay --cloud-secrets --env-file .env",
|
|
1748
|
+
*(
|
|
1749
|
+
[
|
|
1750
|
+
f"Keep bitfab-replay/** branches from triggering {', '.join(triggers)}"
|
|
1751
|
+
]
|
|
1752
|
+
if triggers
|
|
1753
|
+
else []
|
|
1754
|
+
),
|
|
1755
|
+
f"Commit {path} and merge it to the default branch once, so GitHub registers it",
|
|
1756
|
+
"Add --cloud --dry-run to your replay command to check the runner, then replay with --cloud",
|
|
1757
|
+
],
|
|
1846
1758
|
}
|
|
1847
|
-
return {**result, **dropped_command(config)}
|
|
1848
1759
|
|
|
1849
1760
|
|
|
1850
|
-
def
|
|
1851
|
-
|
|
1852
|
-
return {}
|
|
1853
|
-
return {
|
|
1854
|
-
"droppedCommand": config["command"],
|
|
1855
|
-
"droppedCommandNext": "The runner now starts the same bitfab-replay you run locally, from the same directory and with the same arguments, so this command is no longer used. "
|
|
1856
|
-
"If it also set environment variables, add them to the Replay step's env",
|
|
1857
|
-
}
|
|
1761
|
+
def print_json(value):
|
|
1762
|
+
print(json.dumps(value, indent=2), flush=True)
|
|
1858
1763
|
|
|
1859
1764
|
|
|
1860
1765
|
def main():
|
|
1861
1766
|
argv = sys.argv[1:]
|
|
1862
1767
|
try:
|
|
1863
1768
|
if argv[:1] == ["--cloud-init"]:
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1769
|
+
print_json(initialize(argv[1:]))
|
|
1770
|
+
return 0
|
|
1771
|
+
if argv[:1] == ["--cloud-secrets"]:
|
|
1772
|
+
print_json(configure_secrets(argv[1:]))
|
|
1773
|
+
return 0
|
|
1774
|
+
if argv == ["--cloud-execute"]:
|
|
1868
1775
|
result = execute()
|
|
1869
|
-
|
|
1776
|
+
return result["exitCode"] or (1 if result.get("stoppedEarly") else 0)
|
|
1777
|
+
if "-h" in argv or "--help" in argv:
|
|
1870
1778
|
print(HELP, end="")
|
|
1871
1779
|
return 0
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
print(
|
|
1878
|
-
f"Cloud check failed. The Replay step log names what is missing: {result.get('url')}",
|
|
1879
|
-
file=sys.stderr,
|
|
1880
|
-
)
|
|
1881
|
-
elif result.get("testRunId"):
|
|
1882
|
-
print(
|
|
1883
|
-
f"Cloud replay stopped early ({result.get('stoppedEarly', result.get('conclusion'))}). Traces that finished are saved in test run {result['testRunId']}.",
|
|
1884
|
-
file=sys.stderr,
|
|
1885
|
-
)
|
|
1886
|
-
return 1
|
|
1887
|
-
if result.get("state") == "completed":
|
|
1888
|
-
code = report_errored_items(result)
|
|
1889
|
-
if result.get("failOnError") and result.get("errored"):
|
|
1890
|
-
print(
|
|
1891
|
-
"Cloud replay: exiting 1 because of --fail-on-error",
|
|
1892
|
-
file=sys.stderr,
|
|
1893
|
-
)
|
|
1894
|
-
return 1
|
|
1895
|
-
return code
|
|
1896
|
-
return 0
|
|
1780
|
+
record, found = run_cli(argv)
|
|
1781
|
+
if found is not None:
|
|
1782
|
+
return print_outcome(found)
|
|
1783
|
+
print_json(record)
|
|
1784
|
+
return record.get("exitCode", 0) if record.get("state") == "completed" else 0
|
|
1897
1785
|
except KeyboardInterrupt:
|
|
1898
|
-
|
|
1899
|
-
"Detached. The GitHub
|
|
1900
|
-
file=sys.stderr,
|
|
1786
|
+
progress(
|
|
1787
|
+
"Detached. The GitHub run continues; follow it with the printed execution UUID"
|
|
1901
1788
|
)
|
|
1902
1789
|
return 130
|
|
1903
1790
|
except (
|