autonomous-sdlc-harness 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli.js +0 -0
- package/dist/commands/docs.js +2 -1
- package/dist/commands/docs.js.map +1 -1
- package/dist/commands/doctor.js +27 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.js +102 -20
- package/dist/commands/init.js.map +1 -1
- package/dist/config/check.js +11 -4
- package/dist/config/check.js.map +1 -1
- package/dist/config/model.js +25 -0
- package/dist/config/model.js.map +1 -1
- package/dist/core/git.js +29 -0
- package/dist/core/git.js.map +1 -1
- package/dist/core/paths.js +22 -2
- package/dist/core/paths.js.map +1 -1
- package/dist/core/prompt.js +6 -2
- package/dist/core/prompt.js.map +1 -1
- package/dist/core/writer.js +2 -0
- package/dist/core/writer.js.map +1 -1
- package/dist/doctor/checks.js +343 -28
- package/dist/doctor/checks.js.map +1 -1
- package/dist/generators/githubWorkflows.js +66 -0
- package/dist/generators/githubWorkflows.js.map +1 -0
- package/dist/generators/notifications.js +105 -19
- package/dist/generators/notifications.js.map +1 -1
- package/dist/generators/outerLoopScripts.js +17 -3
- package/dist/generators/outerLoopScripts.js.map +1 -1
- package/dist/generators/permissionProfile.js +101 -12
- package/dist/generators/permissionProfile.js.map +1 -1
- package/dist/generators/repoRoot.js +18 -2
- package/dist/generators/repoRoot.js.map +1 -1
- package/dist/generators/stateDir.js +9 -3
- package/dist/generators/stateDir.js.map +1 -1
- package/dist/machine/paths.js +4 -1
- package/dist/machine/paths.js.map +1 -1
- package/dist/remote/githubActions.js +86 -0
- package/dist/remote/githubActions.js.map +1 -0
- package/dist/retrieval/queryLog.js +6 -4
- package/dist/retrieval/queryLog.js.map +1 -1
- package/dist/retrieval/runtime.js +12 -22
- package/dist/retrieval/runtime.js.map +1 -1
- package/dist/retrieval/search.js +21 -10
- package/dist/retrieval/search.js.map +1 -1
- package/dist/retrieval/server.js +1 -1
- package/dist/retrieval/setup.js +2 -1
- package/dist/retrieval/setup.js.map +1 -1
- package/package.json +1 -1
- package/templates/README.md +3 -2
- package/templates/claude/context/conventions.md +1 -1
- package/templates/claude/context/layer.md +1 -1
- package/templates/claude/push-notify.env.example +7 -2
- package/templates/claude/settings.autonomous.json +1 -1
- package/templates/github/workflows/harness-resume.yml +124 -0
- package/templates/github/workflows/harness-run.yml +446 -0
- package/templates/repo/gitignore +5 -0
- package/templates/scripts/README.md +1 -1
- package/templates/scripts/autonomous-watcher.sh +1141 -143
- package/templates/scripts/flow-walker.sh +629 -0
- package/templates/scripts/flows/task_plan_writing.graph.json +192 -0
- package/templates/scripts/lib/flow-walker-gates.sh +165 -0
- package/templates/scripts/lib/harness-run-lib.sh +433 -18
- package/templates/scripts/remote-run.sh +1785 -0
- package/templates/scripts/restart-watcher.sh +21 -1
- package/templates/scripts/run-test-suite.sh +182 -0
- package/templates/scripts/scratch-run.sh +2 -1
- package/templates/state-dir/README-root.md +1 -1
- package/templates/state-dir/autonomous_logs/README.md +1 -1
- package/templates/state-dir/improvement_observations/README.md +2 -0
- package/templates/state-dir/scratch/README.md +1 -1
- package/templates/state-dir/test_fix_plan_reviews/README.md +9 -0
- package/templates/state-dir/test_fix_plans/README.md +9 -0
- package/templates/state-dir/test_fix_point_reviews/README.md +9 -0
- package/templates/state-dir/test_run_logs/README.md +11 -0
|
@@ -0,0 +1,1785 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# remote-run.sh — every call from this machine to GitHub for a harness run: the
|
|
3
|
+
# one place a `gh workflow run` of the run workflow is composed, so the
|
|
4
|
+
# workflow's input contract has exactly one producer on the shell side.
|
|
5
|
+
#
|
|
6
|
+
# THE VERBS AND THE EXIT MAP, stated once for every consumer (the watcher's
|
|
7
|
+
# inbox and relay passes, the job-side `continue` / `poll`, the guard's deny
|
|
8
|
+
# entry and the plugin commands that name this file):
|
|
9
|
+
#
|
|
10
|
+
# remote-run.sh dispatch <branch> --engine <kind> [--resume none|answer|pause]
|
|
11
|
+
# [--answers-from <clar_dir> --indexes "<n> <n>..."]
|
|
12
|
+
# [--park-loop-clear] [--chain <n>] [--repo <root>]
|
|
13
|
+
# remote-run.sh pause <branch> [--repo <root>]
|
|
14
|
+
# remote-run.sh warm [--repo <root>]
|
|
15
|
+
# remote-run.sh stop <branch> [--repo <root>]
|
|
16
|
+
# remote-run.sh status <branch> [--repo <root>]
|
|
17
|
+
# remote-run.sh sync <branch> [--repo <root>]
|
|
18
|
+
# remote-run.sh restore <branch> --resume none|answer|pause [--repo <root>]
|
|
19
|
+
# remote-run.sh save <branch> <out_dir> [--repo <root>]
|
|
20
|
+
# remote-run.sh continue <branch> <bundle_dir> [--repo <root>]
|
|
21
|
+
# remote-run.sh poll [--repo <root>]
|
|
22
|
+
# remote-run.sh pause-requested <branch> <since_epoch> [--repo <root>]
|
|
23
|
+
# remote-run.sh run-created-at <run_id> [--repo <root>]
|
|
24
|
+
# 0 sent (for stop: the action=stop marker was dispatched, and every
|
|
25
|
+
# queued, waiting or in-progress `harness run` run of that branch was
|
|
26
|
+
# asked to cancel, or there was none); for status: printed; for sync: the record is
|
|
27
|
+
# current (including "no run listed yet", which writes nothing); for
|
|
28
|
+
# restore: restored, or no previous bundle (or an expired one, with a
|
|
29
|
+
# `::warning::` line) under --resume none|pause;
|
|
30
|
+
# for save: ALWAYS, whatever happened; for continue: whatever it
|
|
31
|
+
# decided — every outcome a person must act on is a notification; for
|
|
32
|
+
# poll: the tick finished; for pause-requested: such a run exists; for
|
|
33
|
+
# run-created-at: printed
|
|
34
|
+
# 1 usage error, or the library or the configuration could not be
|
|
35
|
+
# resolved; for sync and restore, a local copy or write failed; for
|
|
36
|
+
# pause-requested, also NO such run — a caller that reads 1 as "no
|
|
37
|
+
# pause" passes arguments it has already validated
|
|
38
|
+
# 2 refused, nothing sent or written: execution.target is not
|
|
39
|
+
# github-actions (sending verbs); the branch's local record does not
|
|
40
|
+
# carry `execution: github-actions` (status, sync); the record's mirror
|
|
41
|
+
# working copy is missing, or a downloaded bundle is unrecognised
|
|
42
|
+
# (sync, restore); the inputs payload is over the limit; a named answer
|
|
43
|
+
# file is missing. For restore under --resume answer, "nothing more":
|
|
44
|
+
# no previous bundle, the previous bundle expired (the message names
|
|
45
|
+
# its expiry and the resume command), `HARNESS_INPUT_ANSWERS` not an object of
|
|
46
|
+
# positive-integer keys to strings, or an answer whose `question_<n>.md`
|
|
47
|
+
# is not at the top level of the previous bundle — nothing is restored
|
|
48
|
+
# and no answer is written
|
|
49
|
+
# 3 gh failed: not found, or a non-zero exit — the first line of gh's
|
|
50
|
+
# stderr is named. For poll: the listing or the disable failed. For
|
|
51
|
+
# pause-requested and run-created-at, also an answer that is not the
|
|
52
|
+
# expected JSON; a caller never pauses on a failed read
|
|
53
|
+
#
|
|
54
|
+
# `restore` AND `save` ARE THE JOB-SIDE VERBS: the run workflow calls them in
|
|
55
|
+
# its job, before and (under `always()`) after the harness step. Without
|
|
56
|
+
# `--repo` they act on the checkout of the working directory (`hr_repo_root`),
|
|
57
|
+
# not the main checkout. They test neither `execution.target` nor a registry
|
|
58
|
+
# record: the job exists because a dispatch passed the target gate, and a
|
|
59
|
+
# fresh job checkout carries no registry.
|
|
60
|
+
#
|
|
61
|
+
# WORKFLOW INPUTS REACH THEM THROUGH THE ENVIRONMENT, NEVER A `${{ }}`
|
|
62
|
+
# EXPRESSION INTERPOLATED INTO A SHELL LINE — an input is attacker-shaped text,
|
|
63
|
+
# and interpolation makes it shell source. The workflow sets them with `env:`:
|
|
64
|
+
# HARNESS_INPUT_ANSWERS the `answers` input (restore --resume answer)
|
|
65
|
+
# HARNESS_INPUT_PARK_LOOP_CLEAR the `park_loop_clear` input (restore)
|
|
66
|
+
# `GITHUB_RUN_ID` (restore: this run is never its own previous run) and
|
|
67
|
+
# `GITHUB_STEP_SUMMARY` (save) are the runner's own.
|
|
68
|
+
#
|
|
69
|
+
# `restore` SELECTS the newest `completed` run titled `harness run <branch>`,
|
|
70
|
+
# other than `GITHUB_RUN_ID`, carrying a `harness-state` artifact — walking
|
|
71
|
+
# past a run with none, and stopping at one whose artifact has expired, since
|
|
72
|
+
# an older copy would be staler state. An expired one restores nothing: under
|
|
73
|
+
# --resume answer it exits 2; otherwise it prints a `::warning::` line naming
|
|
74
|
+
# the run, the expiry and the lost counts and clarification history, and the
|
|
75
|
+
# job continues from the committed ledger. An unexpired one it downloads to `<state_dir>/autonomous_logs/remote_download/<branch>/<id>/`
|
|
76
|
+
# (skipped when that directory already holds its status.json); and restores it
|
|
77
|
+
# in `job` mode — on every --resume kind, `none` included, because a reused
|
|
78
|
+
# branch keeps its clarification history. Then, under --resume answer, it
|
|
79
|
+
# writes each `"<n>": "<text>"` entry to `clarifications/<branch>/answer_<n>.md`
|
|
80
|
+
# with the exact bytes, after checking every entry first; and with
|
|
81
|
+
# `HARNESS_INPUT_PARK_LOOP_CLEAR` exactly `true` it sets `park_loop_cycles` to
|
|
82
|
+
# "0" in the restored `autonomous_logs/remote_status.json`. No bundle in any
|
|
83
|
+
# run is an ordinary first job (exit 0, one line) except under --resume answer.
|
|
84
|
+
#
|
|
85
|
+
# `save` WRAPS `hr_remote_bundle_write` into <out_dir>, and with
|
|
86
|
+
# `GITHUB_STEP_SUMMARY` set appends a Markdown table of the bundle's `status`,
|
|
87
|
+
# `decision` and `detail`. With no `autonomous_logs/remote_status.json` and no
|
|
88
|
+
# registry file the harness step never started: <out_dir> is created empty,
|
|
89
|
+
# with no status.json — what `continue` reads as "never started". A
|
|
90
|
+
# `remote_status.json` whose `run_id` is not `GITHUB_RUN_ID` is the previous
|
|
91
|
+
# job's copy that `restore` placed. This job's harness step never wrote its
|
|
92
|
+
# own, so `save` moves it aside to `remote_status.json.previous` and decides as
|
|
93
|
+
# if it were absent. It never
|
|
94
|
+
# fails the job: every problem, a usage error included, is one line on stderr
|
|
95
|
+
# and exit 0.
|
|
96
|
+
#
|
|
97
|
+
# `continue` AND `poll` CLOSE THE LOOP WITHOUT THIS MACHINE, and like `restore`
|
|
98
|
+
# and `save` test neither `execution.target` nor a registry record. `continue`
|
|
99
|
+
# is the run workflow's last job step (under `!cancelled()`); `poll` is the
|
|
100
|
+
# whole body of the resume poller, `WORKFLOW_RESUME_FILE`. Both read, from the
|
|
101
|
+
# environment the workflows set:
|
|
102
|
+
# HARNESS_MAX_CHAIN the automatic-dispatch limit; `24` when empty. Not a
|
|
103
|
+
# non-negative integer: nothing is dispatched
|
|
104
|
+
# HARNESS_POLL_MAX_DISPATCH_FAILURES `poll` only: failed re-dispatches of
|
|
105
|
+
# one paused run before it gives up; `3` when empty
|
|
106
|
+
# HARNESS_POLL_GIVE_UP_AFTER_MINUTES `poll` only: minutes after a run's
|
|
107
|
+
# `usage_resume_at` past which a failed re-dispatch gives
|
|
108
|
+
# up; `360` when empty. Either one not a non-negative
|
|
109
|
+
# integer: nothing is dispatched
|
|
110
|
+
# HARNESS_REMOTE_STOP non-empty: nothing is dispatched
|
|
111
|
+
# HARNESS_REMOTE_SLUG exported as `HARNESS_REPO_SLUG` before a notification,
|
|
112
|
+
# so it names the repository rather than a runner path
|
|
113
|
+
# GITHUB_RUN_ID, GITHUB_SERVER_URL, GITHUB_REPOSITORY the run URL
|
|
114
|
+
# Notifications go through the sibling `autonomous-notify.sh`, as `paused` or
|
|
115
|
+
# `failed`. A re-dispatch is `dispatch <branch> --engine <status.json engine>
|
|
116
|
+
# --resume pause --chain <chain + 1>`, composed by `dispatch` itself.
|
|
117
|
+
#
|
|
118
|
+
# `chain` HAS ONE SOURCE: the bundle's `status.json`, whose `chain` is the
|
|
119
|
+
# writing job's own input — never `HARNESS_INPUT_CHAIN`, the registry or a
|
|
120
|
+
# run's inputs. So `chain + 1` is one more than the job that wrote the bundle,
|
|
121
|
+
# and a user's dispatch (chain 0) restarts the count. An absent or non-integer
|
|
122
|
+
# `chain` is a chain-limit refusal ("chain unreadable"), never 0.
|
|
123
|
+
#
|
|
124
|
+
# `continue <branch> <bundle_dir>` reads the bundle `save` just wrote:
|
|
125
|
+
# no status.json the harness step never started: one `failed` naming the
|
|
126
|
+
# run URL, no dispatch
|
|
127
|
+
# decision continue refused, in this order: `HARNESS_REMOTE_STOP` set (one
|
|
128
|
+
# `paused`); the branch stopped (one log line, no
|
|
129
|
+
# notification — the user asked for it); chain unreadable
|
|
130
|
+
# or `chain + 1` over `HARNESS_MAX_CHAIN` (one `failed`);
|
|
131
|
+
# otherwise re-dispatched. A dispatch that fails is one
|
|
132
|
+
# `paused` naming its error
|
|
133
|
+
# decision wait-poller the branch stopped: one log line. Otherwise `gh
|
|
134
|
+
# workflow enable WORKFLOW_RESUME_FILE`; a failed enable
|
|
135
|
+
# (the job token's enable permission is unverified) is one
|
|
136
|
+
# `paused` saying auto-resume is unavailable
|
|
137
|
+
# decision stop nothing: job mode has already notified
|
|
138
|
+
# A job killed by its step timeout re-dispatches, because job mode writes
|
|
139
|
+
# `decision: continue` the moment it starts; `HARNESS_MAX_CHAIN` is what bounds
|
|
140
|
+
# a job that is killed every time.
|
|
141
|
+
#
|
|
142
|
+
# WHY RE-DISPATCH IS NOT LEFT TO `!cancelled()` ALONE. A user's `stop` cancels a
|
|
143
|
+
# running job, so its `continue` step never runs — but a usage-paused run
|
|
144
|
+
# waiting for the poller has no job to cancel. The stop marker reaches it:
|
|
145
|
+
# `remote_branch_stopped` finds a branch stopped when its newest run titled
|
|
146
|
+
# `harness stop <branch>` was created after its newest `harness run <branch>`,
|
|
147
|
+
# from one bounded newest-first listing of every branch. A marker outside the
|
|
148
|
+
# window is older than every `harness run` inside it, and a user's later
|
|
149
|
+
# dispatch is newer than the marker, so it un-stops the branch with no extra
|
|
150
|
+
# step. It runs ahead of every dispatch and every enable. A listing that fails
|
|
151
|
+
# fails CLOSED in `continue`: nothing sent, one `paused` naming gh's error.
|
|
152
|
+
#
|
|
153
|
+
# `poll` FIRST CARRIES ITS STATE: the newest run of `WORKFLOW_RESUME_FILE` other
|
|
154
|
+
# than `GITHUB_RUN_ID` (bounded) carrying an unexpired `harness-poll-state`
|
|
155
|
+
# artifact is downloaded to `<state_dir>/autonomous_logs/poll_state/previous/`;
|
|
156
|
+
# any failure to find or read it is one line and an empty state. Its
|
|
157
|
+
# `poll_state.json` is {"<branch>": {"run_id", "failures", "notified"}}; an
|
|
158
|
+
# entry whose `run_id` is not the branch's newest `harness run` run is dropped,
|
|
159
|
+
# so a new run restarts the count. `poll` writes the state to `poll_state/
|
|
160
|
+
# current/` on every exit, `HARNESS_REMOTE_STOP` included, and the poller
|
|
161
|
+
# uploads that directory.
|
|
162
|
+
#
|
|
163
|
+
# `poll`: `HARNESS_REMOTE_STOP` set exits 0 with nothing sent. Otherwise one
|
|
164
|
+
# listing; each branch's newest `harness run <branch>` run decides. A run not
|
|
165
|
+
# yet `completed` is never dispatched (its own `continue` will decide): with no
|
|
166
|
+
# `harness-state` artifact it is skipped, not waiting; with one whose bundle
|
|
167
|
+
# says `status: paused` / `pause_reason: usage` it is waiting, because the job
|
|
168
|
+
# uploads before its `continue` step enables the poller; an artifact lookup or
|
|
169
|
+
# download that fails for it is one line and waiting. For a `completed` run,
|
|
170
|
+
# skipped, not waiting: a stopped branch, a bundle that cannot be downloaded
|
|
171
|
+
# (one line), and anything but `status: paused` / `pause_reason: usage` with an
|
|
172
|
+
# integer `usage_resume_at`, and a run whose state entry says `notified`. Due
|
|
173
|
+
# (reset passed): re-dispatched under the same chain limit — a refusal is one
|
|
174
|
+
# `failed` and not waiting; a success drops the branch's state entry. A
|
|
175
|
+
# dispatch that fails counts one more failure for that run and is still
|
|
176
|
+
# waiting, until the count reaches `HARNESS_POLL_MAX_DISPATCH_FAILURES` or the
|
|
177
|
+
# reset is over `HARNESS_POLL_GIVE_UP_AFTER_MINUTES` in the past: then exactly
|
|
178
|
+
# one `paused` naming the dispatch error and the resume command, the entry
|
|
179
|
+
# marked `notified`, and not waiting. Reset ahead: waiting. No branch waiting
|
|
180
|
+
# after the tick: `gh workflow disable WORKFLOW_RESUME_FILE`, then one fresh
|
|
181
|
+
# listing evaluated by the same rules with nothing sent or notified, skipping
|
|
182
|
+
# the branches this tick dispatched — a due run that would be dispatched counts
|
|
183
|
+
# as waiting there. A branch waiting now re-enables the poller, closing the
|
|
184
|
+
# window in which a finishing job enabled it before the disable; a re-listing
|
|
185
|
+
# that fails is one line and the poller stays disabled; a re-enable that fails
|
|
186
|
+
# is one `paused` per waiting branch. Bundles download to
|
|
187
|
+
# `<state_dir>/autonomous_logs/remote_download/<branch>/<id>/` in the checkout,
|
|
188
|
+
# skipped when that directory already holds its status.json.
|
|
189
|
+
#
|
|
190
|
+
# `pause-requested` AND `run-created-at` ARE THE JOB'S TWO READ VERBS, called by
|
|
191
|
+
# `autonomous-watcher.sh job`; like `restore` they test neither
|
|
192
|
+
# `execution.target` nor a registry record, run gh from `hr_repo_root` of the
|
|
193
|
+
# working directory unless `--repo` names one, and write nothing.
|
|
194
|
+
# `pause-requested` lists the branch's runs and exits 0 when one whose
|
|
195
|
+
# `displayTitle` is exactly `harness pause <branch>` has a `createdAt` at or
|
|
196
|
+
# after <since_epoch> — AT, because `createdAt` has one-second resolution and
|
|
197
|
+
# the caller takes <since_epoch> just before the query it will next start
|
|
198
|
+
# from, so a pause created later in that same second is still seen; seeing one
|
|
199
|
+
# twice is harmless, since the caller drops PAUSE once. `run-created-at` prints
|
|
200
|
+
# `gh run view <run_id> --json createdAt` as an epoch second.
|
|
201
|
+
#
|
|
202
|
+
# `status` AND `sync` READ THE RECORD, NOT THE KEY. A run keeps the execution
|
|
203
|
+
# it started with, so they test the record's `execution` field and never
|
|
204
|
+
# `execution.target`; the configuration is still read for `stateDir`.
|
|
205
|
+
#
|
|
206
|
+
# `status` WRITES NOTHING AT ALL — no registry (it does not even create an
|
|
207
|
+
# absent one), no download, no file. It prints the branch's newest runs titled
|
|
208
|
+
# `harness run <branch>` or `harness pause <branch>` (bounded), the record's
|
|
209
|
+
# `status`, `pause_reason`, `remote_run_url` and `remote_synced_at`, whether
|
|
210
|
+
# a `harness run` finished after the last sync, and — reading the newest
|
|
211
|
+
# finished run's artifact list — a line naming its bundle's expiry and the
|
|
212
|
+
# way on when that bundle has expired.
|
|
213
|
+
#
|
|
214
|
+
# `sync` READS THE NEWEST `harness run <branch>` RUN. Any status but
|
|
215
|
+
# `completed` (queued, in_progress, waiting, requested, pending) sets the record
|
|
216
|
+
# `running` and downloads nothing. Otherwise that run — the newest finished one
|
|
217
|
+
# — decides, in exactly one of five cases, tested in this order:
|
|
218
|
+
# 1. its id is the record's `remote_run_id`: already applied. Only
|
|
219
|
+
# `remote_synced_at` is written; nothing is downloaded or restored, so an
|
|
220
|
+
# answer written into the mirror since the last sync survives — unless
|
|
221
|
+
# the record is `parked`, `park_loop` or `paused` (not already `expired`)
|
|
222
|
+
# and that run's bundle has expired since: then as case 2
|
|
223
|
+
# 2. its `harness-state` artifact is listed only as expired: `paused` /
|
|
224
|
+
# `expired`, `remote_run_id` / `remote_run_url` at this run, and
|
|
225
|
+
# `remote_detail` naming the expiry and the way on (resume from the
|
|
226
|
+
# committed ledger, or re-drop the task); nothing restored. The mirror's
|
|
227
|
+
# question files stay, but no job can take an answer to them
|
|
228
|
+
# 3. it carries an unexpired `harness-state` artifact: downloaded (skipped
|
|
229
|
+
# when the download directory already holds its status.json), restored in
|
|
230
|
+
# `mirror` mode into the record's `worktree`, `run.log` copied to the main
|
|
231
|
+
# checkout's `autonomous_logs/<branch>.remote.log`, and `status`,
|
|
232
|
+
# `pause_reason`, `usage_resume_at`, `park_loop_cycles`, `remote_run_id`,
|
|
233
|
+
# `remote_run_url`, `remote_detail` and `remote_synced_at` written
|
|
234
|
+
# 4. no artifact, while some bundle exists (`remote_run_id` is set, or an
|
|
235
|
+
# older finished run carries one): a job that died before its upload.
|
|
236
|
+
# `paused` / `killed`, `remote_run_id` / `remote_run_url` re-pointed at
|
|
237
|
+
# THIS run, nothing restored — so a later sync with no newer run is case 1
|
|
238
|
+
# 5. no bundle in any run and an empty `remote_run_id`: `failed`. Not
|
|
239
|
+
# `paused`: with no bundle anywhere a pause resume has nothing to restore,
|
|
240
|
+
# and re-dropping the artifact is the recovery
|
|
241
|
+
#
|
|
242
|
+
# THE `killed` AND `expired` MAPPINGS. A finished run whose bundle still says
|
|
243
|
+
# `running` (a kill, a timeout with no chain left) syncs as `status: paused`,
|
|
244
|
+
# `pause_reason: killed`; one whose bundle has expired syncs as `status:
|
|
245
|
+
# paused`, `pause_reason: expired`. Both are registry-only — `status.json`
|
|
246
|
+
# never carries either, because `sync` derives them from the run and its
|
|
247
|
+
# artifact list, never from a bundle. Both are `paused` rather than `failed`
|
|
248
|
+
# because a `failed` record has no resume path, while the ledger on the branch
|
|
249
|
+
# is intact and a resume continues from it.
|
|
250
|
+
#
|
|
251
|
+
# THE WORKFLOW INPUT CONTRACT (the workflow template declares the same inputs):
|
|
252
|
+
#
|
|
253
|
+
# action run | pause | warm | stop
|
|
254
|
+
# branch the run's branch; also the dispatch --ref for run, pause
|
|
255
|
+
# and stop. `warm` sends GitHub's default branch as both
|
|
256
|
+
# engine task | user_review | docs (action=run only)
|
|
257
|
+
# resume none | answer | pause (action=run only)
|
|
258
|
+
# answers {"<n>": "<answer_<n>.md bytes>", ...} (resume=answer only)
|
|
259
|
+
# park_loop_clear true, sent only when --park-loop-clear is given
|
|
260
|
+
# chain automatic dispatches since the last user action; a
|
|
261
|
+
# user's dispatch sends 0 (action=run only)
|
|
262
|
+
#
|
|
263
|
+
# The workflow's `run-name` is `harness <action> <branch>`, and the job-side
|
|
264
|
+
# pause poll and `continue` / `poll` match runs by that title, so the spelling
|
|
265
|
+
# is a wire: `pause` and `stop` send the branch as their `branch` input for
|
|
266
|
+
# exactly that reason.
|
|
267
|
+
#
|
|
268
|
+
# `stop` DOES THREE THINGS, IN THIS ORDER. (1) It ALWAYS dispatches action=stop
|
|
269
|
+
# on the branch — a jobless run titled `harness stop <branch>` that GitHub keeps
|
|
270
|
+
# as the stop marker `continue` and `poll` read. It is first because it is the
|
|
271
|
+
# only part that reaches a usage-paused run waiting on the resume poller, which
|
|
272
|
+
# has no job to cancel; a marker dispatch that fails exits 3 at once, before any
|
|
273
|
+
# cancel. (2) It lists the branch's runs of the workflow and cancels each run
|
|
274
|
+
# titled `harness run <branch>` whose status is `queued`, `in_progress` or
|
|
275
|
+
# `waiting` — never a jobless `harness stop` / `harness pause` marker, which has
|
|
276
|
+
# no job to stop and may complete before its cancel lands — trying every one even
|
|
277
|
+
# after a failure. (3) Only when (1) and (2) all succeeded, and only when a
|
|
278
|
+
# local registry record exists, it writes `remote_stopped_at` and sets `status`
|
|
279
|
+
# to `failed` through `hr_registry_set`; a partial stop leaves the record alone
|
|
280
|
+
# and exits 3, so running `stop` again is the remedy.
|
|
281
|
+
#
|
|
282
|
+
# `warm` dispatches action=warm on GitHub's OWN default branch (`gh repo view
|
|
283
|
+
# --json defaultBranchRef`), which may differ from the configured
|
|
284
|
+
# `defaultBranch`: a cache saved there is the one every branch can restore.
|
|
285
|
+
#
|
|
286
|
+
# WHERE gh RUNS. From the main checkout (`hr_main_repo` of the working
|
|
287
|
+
# directory) unless `--repo <root>` names another, so gh resolves the
|
|
288
|
+
# repository from that checkout's remote. The configuration read is that
|
|
289
|
+
# root's `harness.config.json`.
|
|
290
|
+
#
|
|
291
|
+
# WHAT IT NEVER DOES. It never launches a local session, never writes the
|
|
292
|
+
# inbox, never pushes, and never watches a run it sent. Its only writes are the
|
|
293
|
+
# registry record (`stop`, `sync`) and, for `sync`, the download directory
|
|
294
|
+
# `<state_dir>/autonomous_logs/remote_download/<branch>/<id>/` and
|
|
295
|
+
# `<branch>.remote.log` in the main checkout, plus the mirror restore
|
|
296
|
+
# `hr_remote_bundle_restore` performs in the record's `worktree`; for
|
|
297
|
+
# `restore`, that download directory, the job restore, `answer_<n>.md` and the
|
|
298
|
+
# `park_loop_cycles` rewrite of `remote_status.json`, all in the job's
|
|
299
|
+
# checkout; for `save`, <out_dir> and the step summary; for `poll`, its
|
|
300
|
+
# download directories and `<state_dir>/autonomous_logs/poll_state/previous/`
|
|
301
|
+
# and `current/`. `pause-requested` and `run-created-at` write nothing.
|
|
302
|
+
#
|
|
303
|
+
# MIRRORS OF `cli/src/remote/githubActions.ts`, which owns these names; a
|
|
304
|
+
# rename there is an edit here, byte for byte:
|
|
305
|
+
# WORKFLOW_RUN_FILE mirrors WORKFLOW_RUN_FILE
|
|
306
|
+
# WORKFLOW_RESUME_FILE mirrors WORKFLOW_RESUME_FILE
|
|
307
|
+
# STATE_ARTIFACT_NAME mirrors STATE_ARTIFACT_NAME
|
|
308
|
+
# POLL_STATE_ARTIFACT_NAME mirrors POLL_STATE_ARTIFACT_NAME
|
|
309
|
+
# HARNESS_GH_CLI mirrors GH_CLI_VARIABLE (the binary run as `gh`)
|
|
310
|
+
#
|
|
311
|
+
# `set -u` WITHOUT `-e`: every refusal is reported with its own exit code rather
|
|
312
|
+
# than aborting mid-decision.
|
|
313
|
+
#
|
|
314
|
+
# REPRO — every verb and every refusal, against a throwaway fixture, with gh
|
|
315
|
+
# replaced by a recorder stub (nothing reaches the network):
|
|
316
|
+
#
|
|
317
|
+
# d=$(mktemp -d); git -C "$d" init -q; (cd "$d" && npx autonomous-sdlc-harness init)
|
|
318
|
+
# jq '.execution = {target: "github-actions"}' "$d/harness.config.json" > "$d/c" && mv "$d/c" "$d/harness.config.json"
|
|
319
|
+
# s=$(mktemp -d)/gh; printf '%s\n' '#!/bin/sh' 'echo "$*" >> "$0.log"' \
|
|
320
|
+
# 'case "$1 $2" in "run list") echo "[{\"databaseId\":7,\"displayTitle\":\"harness run feat_x\",\"status\":\"in_progress\"}]";;' \
|
|
321
|
+
# '"repo view") echo "{\"defaultBranchRef\":{\"name\":\"main\"}}";; esac' > "$s"; chmod +x "$s"
|
|
322
|
+
# export HARNESS_GH_CLI="$s"; cd "$d"
|
|
323
|
+
#
|
|
324
|
+
# dispatch bash scripts/remote-run.sh dispatch feat_x --engine task; echo $?
|
|
325
|
+
# -> 0; "$s.log" gains `workflow run harness-run.yml --ref feat_x
|
|
326
|
+
# -f action=run -f branch=feat_x -f engine=task -f resume=none -f chain=0`
|
|
327
|
+
# pause bash scripts/remote-run.sh pause feat_x -> 0, `-f action=pause`
|
|
328
|
+
# warm bash scripts/remote-run.sh warm -> 0, `--ref main -f action=warm`
|
|
329
|
+
# stop bash scripts/remote-run.sh stop feat_x -> 0; the `action=stop`
|
|
330
|
+
# dispatch, then `run list ...`, then `run cancel 7`
|
|
331
|
+
# usage bash scripts/remote-run.sh dispatch feat_x -> 1 (no --engine)
|
|
332
|
+
# no config bash scripts/remote-run.sh pause feat_x --repo /tmp -> 1
|
|
333
|
+
# local jq '.execution.target = "local"' ... then any verb -> 2, log unchanged
|
|
334
|
+
# no answer bash scripts/remote-run.sh dispatch feat_x --engine task --resume answer \
|
|
335
|
+
# --answers-from "$d/sdlc-harness/clarifications/feat_x" --indexes 9 -> 2
|
|
336
|
+
# too big head -c 70000 /dev/zero | tr '\0' a > <clar_dir>/answer_1.md, then
|
|
337
|
+
# --resume answer --answers-from <clar_dir> --indexes 1 -> 2, log unchanged
|
|
338
|
+
# gh fails printf '%s\n' '#!/bin/sh' 'echo "boom" >&2' 'exit 4' > "$s"
|
|
339
|
+
# bash scripts/remote-run.sh pause feat_x -> 3, names `boom`
|
|
340
|
+
# gh absent HARNESS_GH_CLI=/nonexistent bash scripts/remote-run.sh warm -> 3
|
|
341
|
+
#
|
|
342
|
+
# status and sync need a remote record, and a `run list` answer whose runs
|
|
343
|
+
# carry `displayTitle` `harness run feat_x` and a `url`:
|
|
344
|
+
# r=sdlc-harness/autonomous_logs/registry.json; mkdir -p "${r%/*}"
|
|
345
|
+
# printf '{"runs":{"feat_x":{"execution":"github-actions","worktree":"%s"}}}' "$d" > "$r"
|
|
346
|
+
# status bash scripts/remote-run.sh status feat_x -> 0; prints the runs
|
|
347
|
+
# titled `harness run feat_x` / `harness pause feat_x`, the record's
|
|
348
|
+
# fields and the finished-since line; "$r" byte-identical
|
|
349
|
+
# sync with the newest such run `in_progress`: bash scripts/remote-run.sh
|
|
350
|
+
# sync feat_x -> 0; the record is `running`, no `run download` logged
|
|
351
|
+
# no record bash scripts/remote-run.sh sync feat_y -> 2, log unchanged
|
|
352
|
+
# no mirror the record's worktree set to /nonexistent, then sync -> 2, names it
|
|
353
|
+
# a bundle a stub answering `run list` with a completed run, `api
|
|
354
|
+
# repos/{owner}/{repo}/actions/runs/<id>/artifacts` with
|
|
355
|
+
# {"artifacts":[{"name":"harness-state","expired":false}]}, and
|
|
356
|
+
# `run download <id> -n harness-state -D <dir>` by writing a bundle
|
|
357
|
+
# (status.json with schema "1", branch feat_x, status parked) into
|
|
358
|
+
# <dir> -> the record is `parked`, remote_run_id is <id>; sync
|
|
359
|
+
# again -> only remote_synced_at changes, no second download
|
|
360
|
+
#
|
|
361
|
+
# restore and save run in the job's checkout; with that same "a bundle" stub
|
|
362
|
+
# (the bundle also carrying clarifications/feat_x/question_1.md and
|
|
363
|
+
# flow_walker_state) and GITHUB_RUN_ID set to another id:
|
|
364
|
+
# restore bash scripts/remote-run.sh restore feat_x --resume none -> 0; the
|
|
365
|
+
# checkout carries clarifications/feat_x/question_1.md,
|
|
366
|
+
# .flow_walker_state and autonomous_logs/remote_status.json
|
|
367
|
+
# answer HARNESS_INPUT_ANSWERS='{"1":"Use B.\n"}' ... --resume answer -> 0;
|
|
368
|
+
# clarifications/feat_x/answer_1.md holds exactly `Use B.` + newline
|
|
369
|
+
# no question HARNESS_INPUT_ANSWERS='{"2":"x"}' ... --resume answer
|
|
370
|
+
# -> 2, no answer_2.md written
|
|
371
|
+
# clear HARNESS_INPUT_PARK_LOOP_CLEAR=true ... --resume none -> 0;
|
|
372
|
+
# remote_status.json's park_loop_cycles is "0"
|
|
373
|
+
# first job a `run list` answer with no finished run: --resume none -> 0;
|
|
374
|
+
# --resume answer -> 2
|
|
375
|
+
# own run GITHUB_RUN_ID=<the bundle run's id> -> that run is skipped
|
|
376
|
+
# expired the artifact list answering {"artifacts":[{"name":"harness-state",
|
|
377
|
+
# "expired":true,"expires_at":"2026-01-02T00:00:00Z"}]}: --resume
|
|
378
|
+
# pause -> 0, a `::warning::` line, no `run download`, no older
|
|
379
|
+
# bundle restored; --resume answer -> 2, names the expiry and
|
|
380
|
+
# branch-resume; sync -> the record is paused / expired; status
|
|
381
|
+
# -> prints the expired line, "$r" byte-identical
|
|
382
|
+
# save bash scripts/remote-run.sh save feat_x /tmp/b -> 0; /tmp/b holds
|
|
383
|
+
# status.json, clarifications/feat_x/, flow_walker_state (and
|
|
384
|
+
# PAUSE_PROGRESS.md, run.log when present); with
|
|
385
|
+
# GITHUB_STEP_SUMMARY=/tmp/s, /tmp/s gains the status table
|
|
386
|
+
# never started no remote_status.json and no registry: save -> 0, /tmp/b
|
|
387
|
+
# empty
|
|
388
|
+
#
|
|
389
|
+
# continue and poll: point HARNESS_PUSH_CMD at a recorder for notifications;
|
|
390
|
+
# <b> is a bundle directory whose status.json carries schema "1":
|
|
391
|
+
# continue decision continue, chain "3": bash scripts/remote-run.sh continue
|
|
392
|
+
# feat_x <b> -> 0; `run list ...`, then `workflow run ... -f
|
|
393
|
+
# resume=pause -f chain=4`
|
|
394
|
+
# limit HARNESS_MAX_CHAIN=3, same bundle -> 0, no `workflow run`, one
|
|
395
|
+
# `failed`; chain "x" -> the same ("chain unreadable")
|
|
396
|
+
# remote stop HARNESS_REMOTE_STOP=1 -> 0, nothing sent, one `paused`
|
|
397
|
+
# stopped a `run list` answer whose `harness stop feat_x` run is newer than
|
|
398
|
+
# its `harness run feat_x` run -> 0, no `workflow run`, no
|
|
399
|
+
# `workflow enable`, no notification
|
|
400
|
+
# list fails a stub failing `run list` -> 0, nothing sent, one `paused`
|
|
401
|
+
# wait-poller decision wait-poller -> `workflow enable harness-resume.yml`;
|
|
402
|
+
# a stub failing it -> one `paused` naming its stderr
|
|
403
|
+
# no status an empty <b> -> 0, one `failed` naming the run URL
|
|
404
|
+
# poll a `run list` answer with a completed `harness run feat_x` run
|
|
405
|
+
# whose bundle says paused / usage / usage_resume_at 1: bash
|
|
406
|
+
# scripts/remote-run.sh poll -> 0; `run download`, `workflow run
|
|
407
|
+
# ... -f resume=pause`, then `workflow disable harness-resume.yml`;
|
|
408
|
+
# with usage_resume_at far ahead -> no dispatch, no disable
|
|
409
|
+
# no dispatch that due bundle with usage_resume_at a minute ago and a stub
|
|
410
|
+
# failing `workflow run`: three ticks, each serving the previous
|
|
411
|
+
# tick's poll_state/current/ as run <id>'s `harness-poll-state`
|
|
412
|
+
# under `run list --workflow harness-resume.yml` -> ticks 1-2 no
|
|
413
|
+
# notification, no disable; tick 3 one `paused` naming the error
|
|
414
|
+
# and branch-resume, then `workflow disable harness-resume.yml`
|
|
415
|
+
# interleave that due run plus an `in_progress` `harness run feat_y` run
|
|
416
|
+
# with no artifact, and a stub that, once its log holds `workflow
|
|
417
|
+
# disable`, lists `harness-state` for feat_y's run with a
|
|
418
|
+
# paused / usage bundle -> the feat_x dispatch, `workflow disable`,
|
|
419
|
+
# then `workflow enable harness-resume.yml`; no feat_y dispatch
|
|
420
|
+
#
|
|
421
|
+
# the job's reads: a `run list` answer whose run has `displayTitle` `harness
|
|
422
|
+
# pause feat_x` and `createdAt` `2026-01-01T00:00:10Z` (epoch 1767225610):
|
|
423
|
+
# pause-requested bash scripts/remote-run.sh pause-requested feat_x 1767225600
|
|
424
|
+
# -> 0; with 1767225620 -> 1; a stub failing `run list` -> 3
|
|
425
|
+
# run-created-at a stub answering `run view 42 --json createdAt` with
|
|
426
|
+
# {"createdAt":"2026-01-01T00:00:10Z"}: bash scripts/remote-run.sh
|
|
427
|
+
# run-created-at 42 -> prints 1767225610, 0; a failing stub -> 3
|
|
428
|
+
|
|
429
|
+
set -u
|
|
430
|
+
|
|
431
|
+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
432
|
+
hr_lib="$script_dir/lib/harness-run-lib.sh"
|
|
433
|
+
if [ ! -r "$hr_lib" ]; then
|
|
434
|
+
echo "remote-run.sh: cannot read '$hr_lib'" >&2
|
|
435
|
+
exit 1
|
|
436
|
+
fi
|
|
437
|
+
# shellcheck source=lib/harness-run-lib.sh
|
|
438
|
+
. "$hr_lib"
|
|
439
|
+
|
|
440
|
+
WORKFLOW_RUN_FILE='harness-run.yml'
|
|
441
|
+
WORKFLOW_RESUME_FILE='harness-resume.yml'
|
|
442
|
+
STATE_ARTIFACT_NAME='harness-state'
|
|
443
|
+
POLL_STATE_ARTIFACT_NAME='harness-poll-state'
|
|
444
|
+
GH="${HARNESS_GH_CLI:-gh}"
|
|
445
|
+
|
|
446
|
+
# How many runs `status` prints, and how many `run list` returns for status
|
|
447
|
+
# and sync — enough to reach past interleaved `harness pause` runs.
|
|
448
|
+
STATUS_RUNS_SHOWN=10
|
|
449
|
+
RUN_LIST_LIMIT=50
|
|
450
|
+
# The one listing of every branch's runs that the stop marker and `poll` read.
|
|
451
|
+
ALL_RUNS_LIMIT=100
|
|
452
|
+
MAX_CHAIN_DEFAULT=24
|
|
453
|
+
# `poll`'s count bound on one paused run's failed re-dispatches: three ticks (90
|
|
454
|
+
# minutes at the shipped `*/30`) ride out a transient GitHub error, and a
|
|
455
|
+
# persistent one costs at most three billed ticks for that branch.
|
|
456
|
+
POLL_MAX_DISPATCH_FAILURES_DEFAULT=3
|
|
457
|
+
# `poll`'s deadline bound, in minutes after `usage_resume_at`. Stateless, so it
|
|
458
|
+
# ends the retries when the carried count is lost; six hours is past what the
|
|
459
|
+
# count bound reaches at any interval up to two hours.
|
|
460
|
+
POLL_GIVE_UP_AFTER_MINUTES_DEFAULT=360
|
|
461
|
+
# How many of the poller's own runs `poll` searches for the previous tick's
|
|
462
|
+
# state artifact; each one without it costs an artifact lookup.
|
|
463
|
+
POLL_STATE_RUNS_LIMIT=10
|
|
464
|
+
|
|
465
|
+
# GitHub's documented limit on a `workflow_dispatch` inputs payload: "The
|
|
466
|
+
# maximum payload for inputs is 65,535 characters."
|
|
467
|
+
# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onworkflow_dispatchinputs
|
|
468
|
+
REMOTE_INPUT_PAYLOAD_MAX=65535
|
|
469
|
+
|
|
470
|
+
EXIT_OK=0
|
|
471
|
+
EXIT_USAGE=1
|
|
472
|
+
EXIT_REFUSED=2
|
|
473
|
+
EXIT_GH=3
|
|
474
|
+
# pause-requested only: the read succeeded and found no pause.
|
|
475
|
+
EXIT_NO_PAUSE=1
|
|
476
|
+
|
|
477
|
+
usage() {
|
|
478
|
+
echo "remote-run.sh: $1" >&2
|
|
479
|
+
echo "usage: remote-run.sh dispatch <branch> --engine <task|user_review|docs> [--resume none|answer|pause] [--answers-from <clar_dir> --indexes \"<n> ...\"] [--park-loop-clear] [--chain <n>] [--repo <root>]" >&2
|
|
480
|
+
echo " remote-run.sh pause <branch> [--repo <root>]" >&2
|
|
481
|
+
echo " remote-run.sh warm [--repo <root>]" >&2
|
|
482
|
+
echo " remote-run.sh stop <branch> [--repo <root>]" >&2
|
|
483
|
+
echo " remote-run.sh status <branch> [--repo <root>]" >&2
|
|
484
|
+
echo " remote-run.sh sync <branch> [--repo <root>]" >&2
|
|
485
|
+
echo " remote-run.sh restore <branch> --resume none|answer|pause [--repo <root>]" >&2
|
|
486
|
+
echo " remote-run.sh save <branch> <out_dir> [--repo <root>]" >&2
|
|
487
|
+
echo " remote-run.sh continue <branch> <bundle_dir> [--repo <root>]" >&2
|
|
488
|
+
echo " remote-run.sh poll [--repo <root>]" >&2
|
|
489
|
+
echo " remote-run.sh pause-requested <branch> <since_epoch> [--repo <root>]" >&2
|
|
490
|
+
echo " remote-run.sh run-created-at <run_id> [--repo <root>]" >&2
|
|
491
|
+
[ "${verb-}" != save ] || exit "$EXIT_OK"
|
|
492
|
+
exit "$EXIT_USAGE"
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
# gh_call <args...> — run gh once with a fixed argument vector. Its stdout is
|
|
496
|
+
# left in GH_OUT; on failure GH_ERR holds the first line of its stderr (or a
|
|
497
|
+
# not-found line) and the return is non-zero.
|
|
498
|
+
GH_OUT=""
|
|
499
|
+
GH_ERR=""
|
|
500
|
+
gh_call() {
|
|
501
|
+
local errfile status
|
|
502
|
+
GH_OUT=""
|
|
503
|
+
GH_ERR=""
|
|
504
|
+
if ! command -v "$GH" >/dev/null 2>&1; then
|
|
505
|
+
GH_ERR="gh not found: '$GH'"
|
|
506
|
+
return 127
|
|
507
|
+
fi
|
|
508
|
+
errfile=$(mktemp) || { GH_ERR="mktemp failed"; return 1; }
|
|
509
|
+
GH_OUT=$("$GH" "$@" 2>"$errfile")
|
|
510
|
+
status=$?
|
|
511
|
+
if [ "$status" -ne 0 ]; then
|
|
512
|
+
IFS= read -r GH_ERR <"$errfile" || :
|
|
513
|
+
[ -n "$GH_ERR" ] || GH_ERR="(no stderr)"
|
|
514
|
+
GH_ERR="gh exited $status: $GH_ERR"
|
|
515
|
+
fi
|
|
516
|
+
rm -f "$errfile"
|
|
517
|
+
return "$status"
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
gh_fail() {
|
|
521
|
+
echo "remote-run.sh: $1: $GH_ERR" >&2
|
|
522
|
+
exit "$EXIT_GH"
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
valid_branch() {
|
|
526
|
+
case "${1-}" in
|
|
527
|
+
''|-*) return 1 ;;
|
|
528
|
+
esac
|
|
529
|
+
return 0
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
# ---------------------------------------------------------------------------
|
|
533
|
+
# Arguments.
|
|
534
|
+
# ---------------------------------------------------------------------------
|
|
535
|
+
|
|
536
|
+
verb=""
|
|
537
|
+
[ "$#" -ge 1 ] || usage "no verb given"
|
|
538
|
+
verb="$1"
|
|
539
|
+
shift
|
|
540
|
+
case "$verb" in
|
|
541
|
+
dispatch|pause|warm|stop|status|sync|restore|save|continue|poll|pause-requested|run-created-at) ;;
|
|
542
|
+
*) usage "unknown verb '$verb'" ;;
|
|
543
|
+
esac
|
|
544
|
+
|
|
545
|
+
branch=""
|
|
546
|
+
out_dir=""
|
|
547
|
+
bundle_dir=""
|
|
548
|
+
engine=""
|
|
549
|
+
resume="none"
|
|
550
|
+
resume_given=0
|
|
551
|
+
answers_from=""
|
|
552
|
+
indexes=""
|
|
553
|
+
indexes_given=0
|
|
554
|
+
park_loop_clear=0
|
|
555
|
+
chain="0"
|
|
556
|
+
repo_arg=""
|
|
557
|
+
since_arg=""
|
|
558
|
+
run_id_arg=""
|
|
559
|
+
|
|
560
|
+
while [ "$#" -gt 0 ]; do
|
|
561
|
+
case "$1" in
|
|
562
|
+
--repo)
|
|
563
|
+
[ "$#" -ge 2 ] || usage "--repo needs a value"
|
|
564
|
+
repo_arg="$2"; shift 2 ;;
|
|
565
|
+
--resume)
|
|
566
|
+
[ "$verb" = dispatch ] || [ "$verb" = restore ] || usage "$1 is a dispatch or restore option"
|
|
567
|
+
[ "$#" -ge 2 ] || usage "$1 needs a value"
|
|
568
|
+
resume="$2"; resume_given=1; shift 2 ;;
|
|
569
|
+
--engine|--answers-from|--indexes|--chain)
|
|
570
|
+
[ "$verb" = dispatch ] || usage "$1 is a dispatch option"
|
|
571
|
+
[ "$#" -ge 2 ] || usage "$1 needs a value"
|
|
572
|
+
case "$1" in
|
|
573
|
+
--engine) engine="$2" ;;
|
|
574
|
+
--answers-from) answers_from="$2" ;;
|
|
575
|
+
--indexes) indexes="$2"; indexes_given=1 ;;
|
|
576
|
+
--chain) chain="$2" ;;
|
|
577
|
+
esac
|
|
578
|
+
shift 2 ;;
|
|
579
|
+
--park-loop-clear)
|
|
580
|
+
[ "$verb" = dispatch ] || usage "$1 is a dispatch option"
|
|
581
|
+
park_loop_clear=1; shift ;;
|
|
582
|
+
-*)
|
|
583
|
+
usage "unknown option '$1'" ;;
|
|
584
|
+
*)
|
|
585
|
+
[ "$verb" != warm ] && [ "$verb" != poll ] || usage "$verb takes no branch"
|
|
586
|
+
if [ "$verb" = run-created-at ]; then
|
|
587
|
+
[ -z "$run_id_arg" ] || usage "unexpected argument '$1'"
|
|
588
|
+
run_id_arg="$1"
|
|
589
|
+
elif [ -z "$branch" ]; then
|
|
590
|
+
branch="$1"
|
|
591
|
+
elif [ "$verb" = pause-requested ] && [ -z "$since_arg" ]; then
|
|
592
|
+
since_arg="$1"
|
|
593
|
+
elif [ "$verb" = save ] && [ -z "$out_dir" ]; then
|
|
594
|
+
out_dir="$1"
|
|
595
|
+
elif [ "$verb" = continue ] && [ -z "$bundle_dir" ]; then
|
|
596
|
+
bundle_dir="$1"
|
|
597
|
+
else
|
|
598
|
+
usage "unexpected argument '$1'"
|
|
599
|
+
fi
|
|
600
|
+
shift ;;
|
|
601
|
+
esac
|
|
602
|
+
done
|
|
603
|
+
|
|
604
|
+
if [ "$verb" != warm ] && [ "$verb" != poll ] && [ "$verb" != run-created-at ]; then
|
|
605
|
+
valid_branch "$branch" || usage "$verb needs a <branch>"
|
|
606
|
+
fi
|
|
607
|
+
|
|
608
|
+
if [ "$verb" = pause-requested ]; then
|
|
609
|
+
case "$since_arg" in
|
|
610
|
+
''|*[!0-9]*) usage "pause-requested needs a <since_epoch> that is a non-negative integer" ;;
|
|
611
|
+
esac
|
|
612
|
+
# Base 10, so a leading zero is neither octal nor invalid JSON for --argjson.
|
|
613
|
+
since_arg=$((10#$since_arg))
|
|
614
|
+
fi
|
|
615
|
+
|
|
616
|
+
if [ "$verb" = run-created-at ]; then
|
|
617
|
+
case "$run_id_arg" in
|
|
618
|
+
''|*[!0-9]*|0*) usage "run-created-at needs a <run_id> that is a positive integer" ;;
|
|
619
|
+
esac
|
|
620
|
+
fi
|
|
621
|
+
|
|
622
|
+
if [ "$verb" = continue ] && [ -z "$bundle_dir" ]; then
|
|
623
|
+
usage "continue needs a <bundle_dir>"
|
|
624
|
+
fi
|
|
625
|
+
|
|
626
|
+
if [ "$verb" = save ] && [ -z "$out_dir" ]; then
|
|
627
|
+
usage "save needs an <out_dir>"
|
|
628
|
+
fi
|
|
629
|
+
|
|
630
|
+
if [ "$verb" = restore ]; then
|
|
631
|
+
[ "$resume_given" -eq 1 ] || usage "restore needs --resume"
|
|
632
|
+
case "$resume" in
|
|
633
|
+
none|answer|pause) ;;
|
|
634
|
+
*) usage "unknown --resume '$resume'" ;;
|
|
635
|
+
esac
|
|
636
|
+
fi
|
|
637
|
+
|
|
638
|
+
if [ "$verb" = dispatch ]; then
|
|
639
|
+
case "$engine" in
|
|
640
|
+
task|user_review|docs) ;;
|
|
641
|
+
'') usage "dispatch needs --engine" ;;
|
|
642
|
+
*) usage "unknown --engine '$engine'" ;;
|
|
643
|
+
esac
|
|
644
|
+
case "$resume" in
|
|
645
|
+
none|answer|pause) ;;
|
|
646
|
+
*) usage "unknown --resume '$resume'" ;;
|
|
647
|
+
esac
|
|
648
|
+
case "$chain" in
|
|
649
|
+
''|*[!0-9]*) usage "--chain must be a non-negative integer" ;;
|
|
650
|
+
esac
|
|
651
|
+
if [ "$resume" = answer ]; then
|
|
652
|
+
[ -n "$answers_from" ] && [ "$indexes_given" -eq 1 ] \
|
|
653
|
+
|| usage "--resume answer needs --answers-from and --indexes"
|
|
654
|
+
[ -n "${indexes// /}" ] || usage "--indexes names no index"
|
|
655
|
+
for n in $indexes; do
|
|
656
|
+
case "$n" in
|
|
657
|
+
''|*[!0-9]*|0*) usage "--indexes must be positive integers, got '$n'" ;;
|
|
658
|
+
esac
|
|
659
|
+
done
|
|
660
|
+
elif [ -n "$answers_from" ] || [ "$indexes_given" -eq 1 ]; then
|
|
661
|
+
usage "--answers-from and --indexes belong to --resume answer"
|
|
662
|
+
fi
|
|
663
|
+
fi
|
|
664
|
+
|
|
665
|
+
# ---------------------------------------------------------------------------
|
|
666
|
+
# The repository and its configuration.
|
|
667
|
+
# ---------------------------------------------------------------------------
|
|
668
|
+
|
|
669
|
+
# setup_fail <message> — a configuration problem: exit 1, except for save.
|
|
670
|
+
setup_fail() {
|
|
671
|
+
echo "remote-run.sh: $1" >&2
|
|
672
|
+
[ "$verb" != save ] || exit "$EXIT_OK"
|
|
673
|
+
exit "$EXIT_USAGE"
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
if [ -n "$repo_arg" ]; then
|
|
677
|
+
root=$(hr_repo_root "$repo_arg") || setup_fail "'$repo_arg' is not a git repository"
|
|
678
|
+
elif [ "$verb" = restore ] || [ "$verb" = save ] || [ "$verb" = continue ] || [ "$verb" = poll ] \
|
|
679
|
+
|| [ "$verb" = pause-requested ] || [ "$verb" = run-created-at ]; then
|
|
680
|
+
root=$(hr_repo_root "${PWD-.}") || setup_fail "'${PWD-.}' is not inside a git repository"
|
|
681
|
+
else
|
|
682
|
+
root=$(hr_main_repo "${PWD-.}") || setup_fail "'${PWD-.}' is not inside a git repository"
|
|
683
|
+
fi
|
|
684
|
+
|
|
685
|
+
hr_config_load "$root" || :
|
|
686
|
+
registry=""
|
|
687
|
+
case "$verb" in
|
|
688
|
+
restore|save|continue|poll)
|
|
689
|
+
hr_state_path "$root" >/dev/null || setup_fail "cannot resolve '$root/harness.config.json'"
|
|
690
|
+
;;
|
|
691
|
+
pause-requested|run-created-at)
|
|
692
|
+
# Read verbs: no gate, and nothing of the configuration is read.
|
|
693
|
+
;;
|
|
694
|
+
status|sync)
|
|
695
|
+
registry=$(hr_state_path "$root" autonomous_logs/registry.json) || {
|
|
696
|
+
echo "remote-run.sh: cannot resolve '$root/harness.config.json'" >&2
|
|
697
|
+
exit "$EXIT_USAGE"
|
|
698
|
+
}
|
|
699
|
+
# Tested with -f first: hr_registry_get creates an absent registry, and
|
|
700
|
+
# status writes nothing.
|
|
701
|
+
if [ ! -f "$registry" ] || [ "$(hr_registry_get "$registry" "$branch" execution)" != github-actions ]; then
|
|
702
|
+
echo "remote-run.sh: refused, nothing written: the local record of '$branch' does not carry execution: github-actions" >&2
|
|
703
|
+
exit "$EXIT_REFUSED"
|
|
704
|
+
fi
|
|
705
|
+
;;
|
|
706
|
+
*)
|
|
707
|
+
target=$(hr_execution_target "$root") || {
|
|
708
|
+
echo "remote-run.sh: cannot resolve '$root/harness.config.json' (or execution.target is outside its enum)" >&2
|
|
709
|
+
exit "$EXIT_USAGE"
|
|
710
|
+
}
|
|
711
|
+
if [ "$target" != github-actions ]; then
|
|
712
|
+
echo "remote-run.sh: refused, nothing sent: execution.target is '$target', not github-actions" >&2
|
|
713
|
+
exit "$EXIT_REFUSED"
|
|
714
|
+
fi
|
|
715
|
+
;;
|
|
716
|
+
esac
|
|
717
|
+
|
|
718
|
+
# Resolved against the caller's directory before the `cd` below.
|
|
719
|
+
case "$bundle_dir" in
|
|
720
|
+
''|/*) ;;
|
|
721
|
+
*) bundle_dir="${PWD-.}/$bundle_dir" ;;
|
|
722
|
+
esac
|
|
723
|
+
|
|
724
|
+
cd "$root" || setup_fail "cannot enter '$root'"
|
|
725
|
+
|
|
726
|
+
# ---------------------------------------------------------------------------
|
|
727
|
+
# The verbs.
|
|
728
|
+
# ---------------------------------------------------------------------------
|
|
729
|
+
|
|
730
|
+
verb_dispatch() {
|
|
731
|
+
local answers="" value n file payload
|
|
732
|
+
local -a inputs
|
|
733
|
+
inputs=(-f "action=run" -f "branch=$branch" -f "engine=$engine" -f "resume=$resume")
|
|
734
|
+
|
|
735
|
+
if [ "$resume" = answer ]; then
|
|
736
|
+
answers='{}'
|
|
737
|
+
for n in $indexes; do
|
|
738
|
+
file="${answers_from%/}/answer_$n.md"
|
|
739
|
+
if [ ! -f "$file" ] || [ ! -r "$file" ]; then
|
|
740
|
+
echo "remote-run.sh: refused, nothing sent: answer file '$file' is missing" >&2
|
|
741
|
+
exit "$EXIT_REFUSED"
|
|
742
|
+
fi
|
|
743
|
+
value=$(jq -R -s . <"$file") || { echo "remote-run.sh: cannot read '$file' as text" >&2; exit "$EXIT_USAGE"; }
|
|
744
|
+
answers=$(jq -n -c --argjson acc "$answers" --arg k "$n" --argjson v "$value" '$acc + {($k): $v}') \
|
|
745
|
+
|| { echo "remote-run.sh: cannot build the answers payload" >&2; exit "$EXIT_USAGE"; }
|
|
746
|
+
done
|
|
747
|
+
inputs+=(-f "answers=$answers")
|
|
748
|
+
fi
|
|
749
|
+
if [ "$park_loop_clear" -eq 1 ]; then
|
|
750
|
+
inputs+=(-f "park_loop_clear=true")
|
|
751
|
+
fi
|
|
752
|
+
inputs+=(-f "chain=$chain")
|
|
753
|
+
|
|
754
|
+
# The limit is on the whole inputs object, so that is what is measured.
|
|
755
|
+
payload=$(jq -n -c --arg action run --arg branch "$branch" --arg engine "$engine" \
|
|
756
|
+
--arg resume "$resume" --arg answers "$answers" --arg plc "$park_loop_clear" --arg chain "$chain" \
|
|
757
|
+
'{action: $action, branch: $branch, engine: $engine, resume: $resume}
|
|
758
|
+
+ (if $answers == "" then {} else {answers: $answers} end)
|
|
759
|
+
+ (if $plc == "1" then {park_loop_clear: "true"} else {} end)
|
|
760
|
+
+ {chain: $chain}') || { echo "remote-run.sh: cannot measure the inputs payload" >&2; exit "$EXIT_USAGE"; }
|
|
761
|
+
if [ "${#payload}" -gt "$REMOTE_INPUT_PAYLOAD_MAX" ]; then
|
|
762
|
+
echo "remote-run.sh: refused, nothing sent: the inputs payload is ${#payload} characters, over GitHub's workflow_dispatch limit of $REMOTE_INPUT_PAYLOAD_MAX" >&2
|
|
763
|
+
exit "$EXIT_REFUSED"
|
|
764
|
+
fi
|
|
765
|
+
|
|
766
|
+
gh_call workflow run "$WORKFLOW_RUN_FILE" --ref "$branch" "${inputs[@]}" || gh_fail "dispatch of '$branch' failed"
|
|
767
|
+
echo "remote-run.sh: dispatched action=run engine=$engine resume=$resume for $branch"
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
verb_pause() {
|
|
771
|
+
gh_call workflow run "$WORKFLOW_RUN_FILE" --ref "$branch" -f "action=pause" -f "branch=$branch" || gh_fail "pause of '$branch' failed"
|
|
772
|
+
echo "remote-run.sh: dispatched action=pause for $branch"
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
verb_warm() {
|
|
776
|
+
local default_branch
|
|
777
|
+
gh_call repo view --json defaultBranchRef || gh_fail "reading GitHub's default branch failed"
|
|
778
|
+
default_branch=$(printf '%s' "$GH_OUT" | jq -r '.defaultBranchRef.name // empty' 2>/dev/null)
|
|
779
|
+
if ! valid_branch "$default_branch"; then
|
|
780
|
+
GH_ERR="no defaultBranchRef.name in its output"
|
|
781
|
+
gh_fail "reading GitHub's default branch failed"
|
|
782
|
+
fi
|
|
783
|
+
gh_call workflow run "$WORKFLOW_RUN_FILE" --ref "$default_branch" -f "action=warm" -f "branch=$default_branch" || gh_fail "warm-up on '$default_branch' failed"
|
|
784
|
+
echo "remote-run.sh: dispatched action=warm on $default_branch"
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
verb_stop() {
|
|
788
|
+
local ids id failed=0 first_err="" registry stopped_at
|
|
789
|
+
stopped_at=$(date +%s)
|
|
790
|
+
gh_call workflow run "$WORKFLOW_RUN_FILE" --ref "$branch" -f "action=stop" -f "branch=$branch" || gh_fail "stop marker for '$branch' failed, nothing cancelled"
|
|
791
|
+
echo "remote-run.sh: dispatched the action=stop marker for $branch"
|
|
792
|
+
|
|
793
|
+
gh_call run list --workflow "$WORKFLOW_RUN_FILE" --branch "$branch" --json databaseId,displayTitle,status --limit 100 || gh_fail "listing the runs of '$branch' failed"
|
|
794
|
+
ids=$(printf '%s' "$GH_OUT" | jq -r --arg t "harness run $branch" '.[] | select(.displayTitle == $t and (.status == "queued" or .status == "in_progress" or .status == "waiting")) | .databaseId' 2>/dev/null) || {
|
|
795
|
+
GH_ERR="its run list is not the expected JSON"
|
|
796
|
+
gh_fail "listing the runs of '$branch' failed"
|
|
797
|
+
}
|
|
798
|
+
for id in $ids; do
|
|
799
|
+
if gh_call run cancel "$id"; then
|
|
800
|
+
echo "remote-run.sh: asked GitHub to cancel run $id of $branch"
|
|
801
|
+
else
|
|
802
|
+
failed=1
|
|
803
|
+
[ -n "$first_err" ] || first_err="$GH_ERR"
|
|
804
|
+
echo "remote-run.sh: cancelling run $id of $branch failed: $GH_ERR" >&2
|
|
805
|
+
fi
|
|
806
|
+
done
|
|
807
|
+
if [ "$failed" -eq 1 ]; then
|
|
808
|
+
GH_ERR="$first_err"
|
|
809
|
+
gh_fail "stop of '$branch' is partial; the local record is unchanged, run stop again"
|
|
810
|
+
fi
|
|
811
|
+
|
|
812
|
+
registry=$(hr_state_path "$root" autonomous_logs/registry.json) || registry=""
|
|
813
|
+
if [ -n "$registry" ] && [ -f "$registry" ] && [ -n "$(hr_registry_get "$registry" "$branch" branch)" ]; then
|
|
814
|
+
hr_registry_set "$registry" "$branch" remote_stopped_at "$stopped_at" \
|
|
815
|
+
&& hr_registry_set "$registry" "$branch" status failed \
|
|
816
|
+
|| echo "remote-run.sh: stopped on GitHub, but the local record of $branch could not be updated" >&2
|
|
817
|
+
fi
|
|
818
|
+
echo "remote-run.sh: stopped $branch"
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
# list_runs — the branch's runs of the workflow into GH_OUT; exits 3 on failure.
|
|
822
|
+
list_runs() {
|
|
823
|
+
gh_call run list --workflow "$WORKFLOW_RUN_FILE" --branch "$branch" \
|
|
824
|
+
--json databaseId,displayTitle,status,conclusion,createdAt,url --limit "$RUN_LIST_LIMIT" \
|
|
825
|
+
|| gh_fail "listing the runs of '$branch' failed"
|
|
826
|
+
printf '%s' "$GH_OUT" | jq -e 'type == "array"' >/dev/null 2>&1 || {
|
|
827
|
+
GH_ERR="its run list is not the expected JSON"
|
|
828
|
+
gh_fail "listing the runs of '$branch' failed"
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
# titled_runs <title> [<title>] — GH_OUT's runs carrying either title, newest
|
|
833
|
+
# first. Two named --arg values rather than --args, which needs jq 1.6.
|
|
834
|
+
titled_runs() {
|
|
835
|
+
printf '%s' "$GH_OUT" | jq -c --arg a "$1" --arg b "${2-$1}" '
|
|
836
|
+
[.[] | select(.displayTitle == $a or .displayTitle == $b)]
|
|
837
|
+
| sort_by([.createdAt, .databaseId]) | reverse'
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
# bundle_listed <run_id> [<artifact_name>] — 0 when the run carries an unexpired
|
|
841
|
+
# artifact of that name (the state artifact when omitted), 1 when it does not, 2
|
|
842
|
+
# with GH_ERR set when the lookup failed. Never exits.
|
|
843
|
+
bundle_listed() {
|
|
844
|
+
local count
|
|
845
|
+
gh_call api "repos/{owner}/{repo}/actions/runs/$1/artifacts" || return 2
|
|
846
|
+
count=$(printf '%s' "$GH_OUT" | jq --arg n "${2:-$STATE_ARTIFACT_NAME}" \
|
|
847
|
+
'[.artifacts[]? | select(.name == $n and (.expired != true))] | length' 2>/dev/null)
|
|
848
|
+
case "$count" in
|
|
849
|
+
''|*[!0-9]*) GH_ERR="its artifact list is not the expected JSON"; return 2 ;;
|
|
850
|
+
esac
|
|
851
|
+
[ "$count" -gt 0 ]
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
# has_bundle <run_id> — bundle_listed, exiting 3 when the lookup failed.
|
|
855
|
+
has_bundle() {
|
|
856
|
+
bundle_listed "$1"
|
|
857
|
+
case $? in
|
|
858
|
+
0) return 0 ;;
|
|
859
|
+
1) return 1 ;;
|
|
860
|
+
esac
|
|
861
|
+
gh_fail "reading the artifacts of run $1 failed"
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
# bundle_state <run_id> — BUNDLE_STATE is `present` (an unexpired state
|
|
865
|
+
# artifact is listed), `expired` (only expired copies are) or `none`; when
|
|
866
|
+
# expired, BUNDLE_EXPIRES_AT is the latest listed `expires_at`. Exits 3 when
|
|
867
|
+
# the lookup failed, as has_bundle does. has_bundle alone cannot tell expired
|
|
868
|
+
# from absent: it reads both as "no bundle".
|
|
869
|
+
BUNDLE_STATE=""
|
|
870
|
+
BUNDLE_EXPIRES_AT=""
|
|
871
|
+
bundle_state() {
|
|
872
|
+
local answer
|
|
873
|
+
BUNDLE_STATE=""
|
|
874
|
+
BUNDLE_EXPIRES_AT=""
|
|
875
|
+
gh_call api "repos/{owner}/{repo}/actions/runs/$1/artifacts" || gh_fail "reading the artifacts of run $1 failed"
|
|
876
|
+
answer=$(printf '%s' "$GH_OUT" | jq -r --arg n "$STATE_ARTIFACT_NAME" '
|
|
877
|
+
[.artifacts[]? | select(.name == $n)] as $a
|
|
878
|
+
| if ($a | map(select(.expired != true)) | length) > 0 then "present"
|
|
879
|
+
elif ($a | length) > 0 then "expired\t" + ([$a[] | .expires_at // empty | tostring] | sort | last // "")
|
|
880
|
+
else "none" end' 2>/dev/null)
|
|
881
|
+
case "$answer" in
|
|
882
|
+
present|none) BUNDLE_STATE="$answer" ;;
|
|
883
|
+
expired$'\t'*)
|
|
884
|
+
BUNDLE_STATE=expired
|
|
885
|
+
BUNDLE_EXPIRES_AT="${answer#*$'\t'}"
|
|
886
|
+
[ -n "$BUNDLE_EXPIRES_AT" ] || BUNDLE_EXPIRES_AT="an unlisted date"
|
|
887
|
+
;;
|
|
888
|
+
*)
|
|
889
|
+
GH_ERR="its artifact list is not the expected JSON"
|
|
890
|
+
gh_fail "reading the artifacts of run $1 failed"
|
|
891
|
+
;;
|
|
892
|
+
esac
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
# expired_line <run_id> — the way on for a bundle bundle_state found expired.
|
|
896
|
+
expired_line() {
|
|
897
|
+
printf '%s' "the state bundle of run $1 expired on $BUNDLE_EXPIRES_AT: resume from the committed ledger with $RESUME_HINT $branch, or re-drop the task"
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
set_or_fail() {
|
|
901
|
+
hr_registry_set "$registry" "$branch" "$1" "$2" || {
|
|
902
|
+
echo "remote-run.sh: writing $1 of $branch to '$registry' failed" >&2
|
|
903
|
+
exit "$EXIT_USAGE"
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
verb_status() {
|
|
908
|
+
local runs finished_id synced_id field
|
|
909
|
+
list_runs
|
|
910
|
+
runs=$(titled_runs "harness run $branch" "harness pause $branch") || runs='[]'
|
|
911
|
+
echo "remote-run.sh: runs of $WORKFLOW_RUN_FILE for $branch, newest first:"
|
|
912
|
+
printf '%s' "$runs" | jq -r --argjson n "$STATUS_RUNS_SHOWN" '
|
|
913
|
+
if length == 0 then " (none)" else
|
|
914
|
+
.[:$n][] | " \(.databaseId) \(.displayTitle) \(.status)/\(.conclusion // "") \(.createdAt) \(.url)" end'
|
|
915
|
+
echo "remote-run.sh: local record (last synced):"
|
|
916
|
+
for field in status pause_reason remote_run_url remote_synced_at; do
|
|
917
|
+
printf ' %s: %s\n' "$field" "$(hr_registry_get "$registry" "$branch" "$field")"
|
|
918
|
+
done
|
|
919
|
+
finished_id=$(printf '%s' "$runs" | jq -r --arg t "harness run $branch" \
|
|
920
|
+
'[.[] | select(.displayTitle == $t and .status == "completed")][0].databaseId // empty | tostring')
|
|
921
|
+
synced_id=$(hr_registry_get "$registry" "$branch" remote_run_id)
|
|
922
|
+
if [ -n "$finished_id" ] && [ "$finished_id" != "$synced_id" ]; then
|
|
923
|
+
echo "remote-run.sh: run $finished_id finished after the last sync; sync would change the record"
|
|
924
|
+
else
|
|
925
|
+
echo "remote-run.sh: no run finished after the last sync"
|
|
926
|
+
fi
|
|
927
|
+
if [ -n "$finished_id" ]; then
|
|
928
|
+
bundle_state "$finished_id"
|
|
929
|
+
[ "$BUNDLE_STATE" != expired ] || echo "remote-run.sh: $(expired_line "$finished_id")"
|
|
930
|
+
fi
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
# sync_expired <run_id> <url> <now> — the record `paused` / `expired` at that
|
|
934
|
+
# run, from BUNDLE_EXPIRES_AT; nothing restored.
|
|
935
|
+
sync_expired() {
|
|
936
|
+
local line
|
|
937
|
+
line=$(expired_line "$1")
|
|
938
|
+
set_or_fail status paused
|
|
939
|
+
set_or_fail pause_reason expired
|
|
940
|
+
set_or_fail remote_run_id "$1"
|
|
941
|
+
set_or_fail remote_run_url "$2"
|
|
942
|
+
set_or_fail remote_detail "$line"
|
|
943
|
+
set_or_fail remote_synced_at "$3"
|
|
944
|
+
echo "remote-run.sh: $line"
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
verb_sync() {
|
|
948
|
+
local worktree runs newest id state url synced_id now older download status_file
|
|
949
|
+
local status reason detail
|
|
950
|
+
worktree=$(hr_registry_get "$registry" "$branch" worktree)
|
|
951
|
+
if [ -z "$worktree" ] || [ ! -d "$worktree" ]; then
|
|
952
|
+
echo "remote-run.sh: refused, nothing written: the mirror working copy '$worktree' of $branch is missing" >&2
|
|
953
|
+
exit "$EXIT_REFUSED"
|
|
954
|
+
fi
|
|
955
|
+
list_runs
|
|
956
|
+
runs=$(titled_runs "harness run $branch") || runs='[]'
|
|
957
|
+
newest=$(printf '%s' "$runs" | jq -c '.[0] // empty')
|
|
958
|
+
if [ -z "$newest" ]; then
|
|
959
|
+
echo "remote-run.sh: no run titled 'harness run $branch' is listed yet; the record is unchanged"
|
|
960
|
+
return 0
|
|
961
|
+
fi
|
|
962
|
+
id=$(printf '%s' "$newest" | jq -r '.databaseId | tostring')
|
|
963
|
+
state=$(printf '%s' "$newest" | jq -r '.status // ""')
|
|
964
|
+
url=$(printf '%s' "$newest" | jq -r '.url // ""')
|
|
965
|
+
now=$(date +%s)
|
|
966
|
+
|
|
967
|
+
if [ "$state" != completed ]; then
|
|
968
|
+
set_or_fail status running
|
|
969
|
+
set_or_fail remote_synced_at "$now"
|
|
970
|
+
echo "remote-run.sh: run $id of $branch is $state; the record is running, nothing downloaded"
|
|
971
|
+
return 0
|
|
972
|
+
fi
|
|
973
|
+
|
|
974
|
+
synced_id=$(hr_registry_get "$registry" "$branch" remote_run_id)
|
|
975
|
+
# Case 1 — already applied. A record still waiting on this run's bundle is
|
|
976
|
+
# re-checked: once it expires, the job can no longer take an answer.
|
|
977
|
+
if [ "$id" = "$synced_id" ]; then
|
|
978
|
+
case "$(hr_registry_get "$registry" "$branch" status)/$(hr_registry_get "$registry" "$branch" pause_reason)" in
|
|
979
|
+
paused/expired) ;;
|
|
980
|
+
parked/*|park_loop/*|paused/*)
|
|
981
|
+
bundle_state "$id"
|
|
982
|
+
if [ "$BUNDLE_STATE" = expired ]; then
|
|
983
|
+
sync_expired "$id" "$url" "$now"
|
|
984
|
+
return 0
|
|
985
|
+
fi
|
|
986
|
+
;;
|
|
987
|
+
esac
|
|
988
|
+
set_or_fail remote_synced_at "$now"
|
|
989
|
+
echo "remote-run.sh: run $id of $branch is the one last synced; the mirror is current"
|
|
990
|
+
return 0
|
|
991
|
+
fi
|
|
992
|
+
|
|
993
|
+
# Case 2 — a newer run whose bundle has expired.
|
|
994
|
+
bundle_state "$id"
|
|
995
|
+
if [ "$BUNDLE_STATE" = expired ]; then
|
|
996
|
+
sync_expired "$id" "$url" "$now"
|
|
997
|
+
return 0
|
|
998
|
+
fi
|
|
999
|
+
|
|
1000
|
+
# Case 3 — a newer run with a bundle.
|
|
1001
|
+
if [ "$BUNDLE_STATE" = present ]; then
|
|
1002
|
+
hr_remote_names_var
|
|
1003
|
+
download=$(hr_state_path "$root" "autonomous_logs/remote_download/$branch/$id") || {
|
|
1004
|
+
echo "remote-run.sh: cannot resolve '$root/harness.config.json'" >&2
|
|
1005
|
+
exit "$EXIT_USAGE"
|
|
1006
|
+
}
|
|
1007
|
+
status_file="$download/$HR_REMOTE_STATUS_FILE"
|
|
1008
|
+
if [ ! -f "$status_file" ]; then
|
|
1009
|
+
mkdir -p "$download" || { echo "remote-run.sh: cannot create '$download'" >&2; exit "$EXIT_USAGE"; }
|
|
1010
|
+
gh_call run download "$id" -n "$STATE_ARTIFACT_NAME" -D "$download" || gh_fail "downloading the bundle of run $id failed"
|
|
1011
|
+
fi
|
|
1012
|
+
status=$(hr_remote_status_get "$status_file" status) || status=""
|
|
1013
|
+
case "$status" in
|
|
1014
|
+
running|parked|park_loop|paused|completed|failed) ;;
|
|
1015
|
+
*)
|
|
1016
|
+
echo "remote-run.sh: refused, nothing written: the bundle in '$download' is unrecognised" >&2
|
|
1017
|
+
exit "$EXIT_REFUSED"
|
|
1018
|
+
;;
|
|
1019
|
+
esac
|
|
1020
|
+
hr_remote_bundle_restore "$download" "$worktree" "$branch" mirror
|
|
1021
|
+
case $? in
|
|
1022
|
+
0) ;;
|
|
1023
|
+
2) echo "remote-run.sh: refused, nothing written: the bundle in '$download' is unrecognised for $branch" >&2; exit "$EXIT_REFUSED" ;;
|
|
1024
|
+
*) echo "remote-run.sh: restoring '$download' into '$worktree' failed" >&2; exit "$EXIT_USAGE" ;;
|
|
1025
|
+
esac
|
|
1026
|
+
if [ -f "$download/$HR_REMOTE_LOG_FILE" ]; then
|
|
1027
|
+
local remote_log
|
|
1028
|
+
remote_log=$(hr_state_path "$root" "autonomous_logs/$branch.remote.log") || remote_log=""
|
|
1029
|
+
if [ -n "$remote_log" ]; then
|
|
1030
|
+
mkdir -p "${remote_log%/*}" && cp "$download/$HR_REMOTE_LOG_FILE" "$remote_log" \
|
|
1031
|
+
|| { echo "remote-run.sh: copying the run log to '$remote_log' failed" >&2; exit "$EXIT_USAGE"; }
|
|
1032
|
+
fi
|
|
1033
|
+
fi
|
|
1034
|
+
reason=$(hr_remote_status_get "$status_file" pause_reason) || reason=""
|
|
1035
|
+
detail=$(hr_remote_status_get "$status_file" detail) || detail=""
|
|
1036
|
+
if [ "$status" = running ]; then
|
|
1037
|
+
status=paused
|
|
1038
|
+
reason=killed
|
|
1039
|
+
detail="the job ended mid-run (its bundle still says running): $url"
|
|
1040
|
+
fi
|
|
1041
|
+
[ -n "$detail" ] || detail="synced from $url"
|
|
1042
|
+
set_or_fail status "$status"
|
|
1043
|
+
set_or_fail pause_reason "$reason"
|
|
1044
|
+
set_or_fail usage_resume_at "$(hr_remote_status_get "$status_file" usage_resume_at || :)"
|
|
1045
|
+
set_or_fail park_loop_cycles "$(hr_remote_status_get "$status_file" park_loop_cycles || :)"
|
|
1046
|
+
set_or_fail remote_run_id "$id"
|
|
1047
|
+
set_or_fail remote_run_url "$url"
|
|
1048
|
+
set_or_fail remote_detail "$detail"
|
|
1049
|
+
set_or_fail remote_synced_at "$now"
|
|
1050
|
+
echo "remote-run.sh: synced run $id of $branch: $status${reason:+ ($reason)}"
|
|
1051
|
+
return 0
|
|
1052
|
+
fi
|
|
1053
|
+
|
|
1054
|
+
# Case 4 — a newer run with no bundle, while some bundle exists.
|
|
1055
|
+
local bundle_exists=0
|
|
1056
|
+
if [ -n "$synced_id" ]; then
|
|
1057
|
+
bundle_exists=1
|
|
1058
|
+
else
|
|
1059
|
+
for older in $(printf '%s' "$runs" | jq -r '.[1:][] | select(.status == "completed") | .databaseId | tostring'); do
|
|
1060
|
+
if has_bundle "$older"; then bundle_exists=1; break; fi
|
|
1061
|
+
done
|
|
1062
|
+
fi
|
|
1063
|
+
if [ "$bundle_exists" -eq 1 ]; then
|
|
1064
|
+
set_or_fail status paused
|
|
1065
|
+
set_or_fail pause_reason killed
|
|
1066
|
+
set_or_fail remote_run_id "$id"
|
|
1067
|
+
set_or_fail remote_run_url "$url"
|
|
1068
|
+
set_or_fail remote_detail "run $id ended with no state bundle (killed, cancelled or replaced): $url"
|
|
1069
|
+
set_or_fail remote_synced_at "$now"
|
|
1070
|
+
echo "remote-run.sh: run $id of $branch left no bundle; the record is paused (killed), nothing restored"
|
|
1071
|
+
return 0
|
|
1072
|
+
fi
|
|
1073
|
+
|
|
1074
|
+
# Case 5 — no bundle in any run.
|
|
1075
|
+
set_or_fail status failed
|
|
1076
|
+
set_or_fail remote_detail "no run of $branch ever uploaded a state bundle; newest: $url"
|
|
1077
|
+
set_or_fail remote_synced_at "$now"
|
|
1078
|
+
echo "remote-run.sh: no run of $branch carries a state bundle; the record is failed"
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
restore_fail() {
|
|
1082
|
+
echo "remote-run.sh: $1" >&2
|
|
1083
|
+
exit "$EXIT_USAGE"
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
restore_refuse() {
|
|
1087
|
+
echo "remote-run.sh: refused: $1" >&2
|
|
1088
|
+
exit "$EXIT_REFUSED"
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
# previous_bundle_run — PREV_RUN_ID is the newest finished `harness run
|
|
1092
|
+
# <branch>` run, other than this job's own, carrying a state artifact, and
|
|
1093
|
+
# PREV_RUN_STATE is `present`, `expired` or empty when no run carries one. A
|
|
1094
|
+
# run with no artifact is walked past; an expired one stops the walk, because
|
|
1095
|
+
# an older copy is staler state. Exits 3 when gh fails.
|
|
1096
|
+
PREV_RUN_ID=""
|
|
1097
|
+
PREV_RUN_STATE=""
|
|
1098
|
+
previous_bundle_run() {
|
|
1099
|
+
local ids id
|
|
1100
|
+
PREV_RUN_ID=""
|
|
1101
|
+
PREV_RUN_STATE=""
|
|
1102
|
+
gh_call run list --workflow "$WORKFLOW_RUN_FILE" --branch "$branch" \
|
|
1103
|
+
--json databaseId,displayTitle,status,createdAt --limit "$RUN_LIST_LIMIT" \
|
|
1104
|
+
|| gh_fail "listing the runs of '$branch' failed"
|
|
1105
|
+
ids=$(printf '%s' "$GH_OUT" | jq -r --arg t "harness run $branch" --arg self "${GITHUB_RUN_ID-}" '
|
|
1106
|
+
[.[] | select(.displayTitle == $t and .status == "completed" and (.databaseId | tostring) != $self)]
|
|
1107
|
+
| sort_by([.createdAt, .databaseId]) | reverse | .[].databaseId | tostring' 2>/dev/null) || {
|
|
1108
|
+
GH_ERR="its run list is not the expected JSON"
|
|
1109
|
+
gh_fail "listing the runs of '$branch' failed"
|
|
1110
|
+
}
|
|
1111
|
+
for id in $ids; do
|
|
1112
|
+
bundle_state "$id"
|
|
1113
|
+
if [ "$BUNDLE_STATE" != none ]; then
|
|
1114
|
+
PREV_RUN_ID="$id"
|
|
1115
|
+
PREV_RUN_STATE="$BUNDLE_STATE"
|
|
1116
|
+
return 0
|
|
1117
|
+
fi
|
|
1118
|
+
done
|
|
1119
|
+
return 0
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
# The answers input: an object of positive-integer keys to strings, read from
|
|
1123
|
+
# the environment by jq itself (`env`, jq 1.5) so no shell word ever holds it.
|
|
1124
|
+
ANSWERS_SHAPE='env.HARNESS_INPUT_ANSWERS | fromjson
|
|
1125
|
+
| type == "object" and length > 0
|
|
1126
|
+
and all(to_entries[]; (.value | type) == "string"
|
|
1127
|
+
and (.key | explode | length > 0 and .[0] != 48 and all(.[]; . >= 48 and . <= 57)))'
|
|
1128
|
+
|
|
1129
|
+
verb_restore() {
|
|
1130
|
+
local id download status_file clar n tmp status_source
|
|
1131
|
+
if [ "$resume" = answer ]; then
|
|
1132
|
+
HARNESS_INPUT_ANSWERS="${HARNESS_INPUT_ANSWERS-}"
|
|
1133
|
+
export HARNESS_INPUT_ANSWERS
|
|
1134
|
+
jq -n -e "$ANSWERS_SHAPE" >/dev/null 2>&1 \
|
|
1135
|
+
|| restore_refuse "HARNESS_INPUT_ANSWERS is not an object of positive-integer keys to strings; nothing written"
|
|
1136
|
+
fi
|
|
1137
|
+
|
|
1138
|
+
previous_bundle_run
|
|
1139
|
+
id="$PREV_RUN_ID"
|
|
1140
|
+
hr_remote_names_var
|
|
1141
|
+
if [ "$PREV_RUN_STATE" = expired ]; then
|
|
1142
|
+
[ "$resume" != answer ] \
|
|
1143
|
+
|| restore_refuse "the state bundle of run $id expired on $BUNDLE_EXPIRES_AT, so its questions can no longer be answered here: resume from the committed ledger with $RESUME_HINT $branch, or re-drop the task; nothing written"
|
|
1144
|
+
echo "::warning::remote-run.sh: the state bundle of run $id expired on $BUNDLE_EXPIRES_AT: the park-loop, auto-resume and stall counts and the clarification history it carried are lost; this job continues from the committed ledger"
|
|
1145
|
+
elif [ -z "$id" ]; then
|
|
1146
|
+
[ "$resume" != answer ] \
|
|
1147
|
+
|| restore_refuse "--resume answer, but no finished run of $branch carries a state bundle; nothing written"
|
|
1148
|
+
echo "remote-run.sh: no previous bundle for $branch; this is its first job"
|
|
1149
|
+
else
|
|
1150
|
+
download=$(hr_state_path "$root" "autonomous_logs/remote_download/$branch/$id") \
|
|
1151
|
+
|| restore_fail "cannot resolve '$root/harness.config.json'"
|
|
1152
|
+
status_file="$download/$HR_REMOTE_STATUS_FILE"
|
|
1153
|
+
if [ ! -f "$status_file" ]; then
|
|
1154
|
+
mkdir -p "$download" || restore_fail "cannot create '$download'"
|
|
1155
|
+
gh_call run download "$id" -n "$STATE_ARTIFACT_NAME" -D "$download" || gh_fail "downloading the bundle of run $id failed"
|
|
1156
|
+
fi
|
|
1157
|
+
if [ "$resume" = answer ]; then
|
|
1158
|
+
# Checked against the downloaded bundle, before anything is restored: a refusal
|
|
1159
|
+
# must leave no restored status for `save` to re-upload as this job's own.
|
|
1160
|
+
for n in $(jq -n -r 'env.HARNESS_INPUT_ANSWERS | fromjson | keys_unsorted[]'); do
|
|
1161
|
+
[ -f "$download/$HR_REMOTE_CLARIFY_DIR/$branch/question_$n.md" ] \
|
|
1162
|
+
|| restore_refuse "answer $n has no question_$n.md in the bundle of run $id; nothing restored, no answer written"
|
|
1163
|
+
done
|
|
1164
|
+
fi
|
|
1165
|
+
hr_remote_bundle_restore "$download" "$root" "$branch" job
|
|
1166
|
+
case $? in
|
|
1167
|
+
0) echo "remote-run.sh: restored the bundle of run $id into $root" ;;
|
|
1168
|
+
2) restore_refuse "the bundle in '$download' is unrecognised for $branch; nothing restored" ;;
|
|
1169
|
+
*) restore_fail "restoring '$download' into '$root' failed" ;;
|
|
1170
|
+
esac
|
|
1171
|
+
fi
|
|
1172
|
+
|
|
1173
|
+
if [ "$resume" = answer ]; then
|
|
1174
|
+
clar=$(hr_state_path "$root" "$HR_REMOTE_CLARIFY_DIR/$branch") \
|
|
1175
|
+
|| restore_fail "cannot resolve '$root/harness.config.json'"
|
|
1176
|
+
for n in $(jq -n -r 'env.HARNESS_INPUT_ANSWERS | fromjson | keys_unsorted[]'); do
|
|
1177
|
+
tmp=$(mktemp "$clar/answer_$n.md.tmp.XXXXXX") || restore_fail "cannot write in '$clar'"
|
|
1178
|
+
if jq -n -j --arg k "$n" 'env.HARNESS_INPUT_ANSWERS | fromjson | .[$k]' >"$tmp" \
|
|
1179
|
+
&& mv "$tmp" "$clar/answer_$n.md"; then
|
|
1180
|
+
echo "remote-run.sh: wrote answer_$n.md for $branch"
|
|
1181
|
+
else
|
|
1182
|
+
rm -f "$tmp"
|
|
1183
|
+
restore_fail "writing '$clar/answer_$n.md' failed"
|
|
1184
|
+
fi
|
|
1185
|
+
done
|
|
1186
|
+
fi
|
|
1187
|
+
|
|
1188
|
+
if [ "${HARNESS_INPUT_PARK_LOOP_CLEAR-}" = true ]; then
|
|
1189
|
+
status_source=$(hr_state_path "$root" "$HR_REMOTE_STATUS_SOURCE") \
|
|
1190
|
+
|| restore_fail "cannot resolve '$root/harness.config.json'"
|
|
1191
|
+
if [ -f "$status_source" ]; then
|
|
1192
|
+
tmp=$(mktemp "$status_source.tmp.XXXXXX") || restore_fail "cannot write beside '$status_source'"
|
|
1193
|
+
if jq '.park_loop_cycles = "0"' "$status_source" >"$tmp" && mv "$tmp" "$status_source"; then
|
|
1194
|
+
echo "remote-run.sh: cleared park_loop_cycles for $branch"
|
|
1195
|
+
else
|
|
1196
|
+
rm -f "$tmp"
|
|
1197
|
+
restore_fail "clearing park_loop_cycles in '$status_source' failed"
|
|
1198
|
+
fi
|
|
1199
|
+
else
|
|
1200
|
+
echo "remote-run.sh: park_loop_clear given, but no restored status to clear for $branch"
|
|
1201
|
+
fi
|
|
1202
|
+
fi
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
# md_cell <text> — one Markdown table cell: pipes escaped, line breaks flattened.
|
|
1206
|
+
md_cell() {
|
|
1207
|
+
local s="${1-}"
|
|
1208
|
+
s=${s//$'\r'/ }
|
|
1209
|
+
s=${s//$'\n'/ }
|
|
1210
|
+
printf '%s' "${s//|/\\|}"
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
verb_save() {
|
|
1214
|
+
local registry_file status_source status decision detail
|
|
1215
|
+
hr_remote_names_var
|
|
1216
|
+
registry_file=$(hr_state_path "$root" autonomous_logs/registry.json) || {
|
|
1217
|
+
echo "remote-run.sh: save: cannot resolve '$root/harness.config.json'; no bundle written" >&2
|
|
1218
|
+
return 0
|
|
1219
|
+
}
|
|
1220
|
+
status_source=$(hr_state_path "$root" "$HR_REMOTE_STATUS_SOURCE") || status_source=""
|
|
1221
|
+
# `restore` places the previous job's status.json at this same path; one whose
|
|
1222
|
+
# run_id is not this job's means this job's harness step never wrote its own,
|
|
1223
|
+
# and re-uploading it would replay the previous job's decision and chain.
|
|
1224
|
+
if [ -n "$status_source" ] && [ -f "$status_source" ] && [ -n "${GITHUB_RUN_ID-}" ] \
|
|
1225
|
+
&& [ "$(hr_remote_status_get "$status_source" run_id 2>/dev/null || :)" != "$GITHUB_RUN_ID" ]; then
|
|
1226
|
+
if mv "$status_source" "$status_source.previous" 2>/dev/null; then
|
|
1227
|
+
echo "remote-run.sh: save: $status_source is the previous job's (run_id is not $GITHUB_RUN_ID); moved aside, not uploaded" >&2
|
|
1228
|
+
else
|
|
1229
|
+
echo "remote-run.sh: save: cannot move the previous job's '$status_source' aside; no bundle written" >&2
|
|
1230
|
+
mkdir -p "$out_dir" 2>/dev/null || :
|
|
1231
|
+
status_source=""
|
|
1232
|
+
registry_file=""
|
|
1233
|
+
fi
|
|
1234
|
+
fi
|
|
1235
|
+
# Without either file the harness step never started; the library's registry
|
|
1236
|
+
# fallback would create a registry in the checkout to find nothing in it.
|
|
1237
|
+
if { [ -z "$status_source" ] || [ ! -f "$status_source" ]; } && { [ -z "$registry_file" ] || [ ! -f "$registry_file" ]; }; then
|
|
1238
|
+
mkdir -p "$out_dir" 2>/dev/null \
|
|
1239
|
+
|| echo "remote-run.sh: save: cannot create '$out_dir'" >&2
|
|
1240
|
+
echo "remote-run.sh: save: the harness step never started for $branch; the bundle carries no status.json" >&2
|
|
1241
|
+
else
|
|
1242
|
+
hr_remote_bundle_write "$root" "$branch" "$registry_file" "$out_dir"
|
|
1243
|
+
case $? in
|
|
1244
|
+
0) echo "remote-run.sh: saved the bundle of $branch into $out_dir" ;;
|
|
1245
|
+
2) echo "remote-run.sh: save: cannot resolve '$root/harness.config.json'; no bundle written" >&2 ;;
|
|
1246
|
+
*)
|
|
1247
|
+
if [ -f "$status_source" ]; then
|
|
1248
|
+
echo "remote-run.sh: save: assembling the bundle in '$out_dir' failed (not empty, or a copy failed)" >&2
|
|
1249
|
+
else
|
|
1250
|
+
echo "remote-run.sh: save: no status for $branch in '$registry_file'; the bundle carries no status.json" >&2
|
|
1251
|
+
fi
|
|
1252
|
+
;;
|
|
1253
|
+
esac
|
|
1254
|
+
fi
|
|
1255
|
+
|
|
1256
|
+
[ -n "${GITHUB_STEP_SUMMARY-}" ] || return 0
|
|
1257
|
+
if [ -f "$out_dir/$HR_REMOTE_STATUS_FILE" ]; then
|
|
1258
|
+
status=$(hr_remote_status_get "$out_dir/$HR_REMOTE_STATUS_FILE" status) || status=""
|
|
1259
|
+
decision=$(hr_remote_status_get "$out_dir/$HR_REMOTE_STATUS_FILE" decision) || decision=""
|
|
1260
|
+
detail=$(hr_remote_status_get "$out_dir/$HR_REMOTE_STATUS_FILE" detail) || detail=""
|
|
1261
|
+
printf '%s\n' "### harness run $(md_cell "$branch")" "" "| status | decision | detail |" "|---|---|---|" \
|
|
1262
|
+
"| $(md_cell "$status") | $(md_cell "$decision") | $(md_cell "$detail") |" "" >>"$GITHUB_STEP_SUMMARY" \
|
|
1263
|
+
|| echo "remote-run.sh: save: cannot append to GITHUB_STEP_SUMMARY" >&2
|
|
1264
|
+
else
|
|
1265
|
+
printf '%s\n' "### harness run $(md_cell "$branch")" "" "No status.json: the harness step never started." "" >>"$GITHUB_STEP_SUMMARY" \
|
|
1266
|
+
|| echo "remote-run.sh: save: cannot append to GITHUB_STEP_SUMMARY" >&2
|
|
1267
|
+
fi
|
|
1268
|
+
return 0
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
# notify <event> <branch> <detail> — one lifecycle notification; never fails.
|
|
1272
|
+
notify() {
|
|
1273
|
+
if [ -n "${HARNESS_REMOTE_SLUG-}" ]; then
|
|
1274
|
+
HARNESS_REPO_SLUG="$HARNESS_REMOTE_SLUG"
|
|
1275
|
+
export HARNESS_REPO_SLUG
|
|
1276
|
+
fi
|
|
1277
|
+
bash "$script_dir/autonomous-notify.sh" "$1" "$2" "" "$3" \
|
|
1278
|
+
|| echo "remote-run.sh: the $1 notification for $2 could not be sent" >&2
|
|
1279
|
+
echo "remote-run.sh: notified $1 for $2: $3"
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
this_run_url() {
|
|
1283
|
+
if [ -n "${GITHUB_RUN_ID-}" ] && [ -n "${GITHUB_SERVER_URL-}" ] && [ -n "${GITHUB_REPOSITORY-}" ]; then
|
|
1284
|
+
printf '%s' "${GITHUB_SERVER_URL%/}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
1285
|
+
else
|
|
1286
|
+
printf 'run %s' "${GITHUB_RUN_ID:-(unknown)}"
|
|
1287
|
+
fi
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
# max_chain_var — MAX_CHAIN from HARNESS_MAX_CHAIN; 1 when it is not a
|
|
1291
|
+
# non-negative integer.
|
|
1292
|
+
MAX_CHAIN=""
|
|
1293
|
+
max_chain_var() {
|
|
1294
|
+
MAX_CHAIN="${HARNESS_MAX_CHAIN:-$MAX_CHAIN_DEFAULT}"
|
|
1295
|
+
case "$MAX_CHAIN" in
|
|
1296
|
+
''|*[!0-9]*) return 1 ;;
|
|
1297
|
+
esac
|
|
1298
|
+
return 0
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
# ALL_RUNS — every branch's runs of the workflow, newest first and bounded,
|
|
1302
|
+
# listed once per invocation. 1 with GH_ERR set when the listing failed.
|
|
1303
|
+
ALL_RUNS=""
|
|
1304
|
+
ALL_RUNS_LISTED=0
|
|
1305
|
+
list_all_runs() {
|
|
1306
|
+
[ "$ALL_RUNS_LISTED" -eq 0 ] || return 0
|
|
1307
|
+
gh_call run list --workflow "$WORKFLOW_RUN_FILE" \
|
|
1308
|
+
--json databaseId,headBranch,displayTitle,status,createdAt --limit "$ALL_RUNS_LIMIT" || return 1
|
|
1309
|
+
printf '%s' "$GH_OUT" | jq -e 'type == "array"' >/dev/null 2>&1 || {
|
|
1310
|
+
GH_ERR="its run list is not the expected JSON"
|
|
1311
|
+
return 1
|
|
1312
|
+
}
|
|
1313
|
+
ALL_RUNS="$GH_OUT"
|
|
1314
|
+
ALL_RUNS_LISTED=1
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
# remote_branch_stopped <branch> — 0 stopped (its newest `harness stop` run is
|
|
1318
|
+
# newer than its newest `harness run` run), 1 not stopped, 2 the listing failed.
|
|
1319
|
+
remote_branch_stopped() {
|
|
1320
|
+
local verdict
|
|
1321
|
+
list_all_runs || return 2
|
|
1322
|
+
verdict=$(printf '%s' "$ALL_RUNS" | jq -r --arg s "harness stop $1" --arg r "harness run $1" '
|
|
1323
|
+
([.[] | select(.displayTitle == $s) | .createdAt // ""] | max // "") as $stop
|
|
1324
|
+
| ([.[] | select(.displayTitle == $r) | .createdAt // ""] | max // "") as $run
|
|
1325
|
+
| if $stop != "" and $stop > $run then "stopped" else "not" end' 2>/dev/null) || {
|
|
1326
|
+
GH_ERR="its run list is not the expected JSON"
|
|
1327
|
+
return 2
|
|
1328
|
+
}
|
|
1329
|
+
[ "$verdict" = stopped ]
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
# redispatch <engine> <chain> — `dispatch <branch> --engine <engine> --resume
|
|
1333
|
+
# pause --chain <chain>` for the global branch, in a subshell so dispatch's own
|
|
1334
|
+
# exits stay its own. On failure REDISPATCH_ERR holds its first stderr line.
|
|
1335
|
+
REDISPATCH_ERR=""
|
|
1336
|
+
redispatch() {
|
|
1337
|
+
local errfile status
|
|
1338
|
+
REDISPATCH_ERR=""
|
|
1339
|
+
errfile=$(mktemp) || { REDISPATCH_ERR="mktemp failed"; return 1; }
|
|
1340
|
+
(
|
|
1341
|
+
engine="$1"; resume=pause; chain="$2"
|
|
1342
|
+
answers_from=""; indexes=""; indexes_given=0; park_loop_clear=0
|
|
1343
|
+
verb_dispatch
|
|
1344
|
+
) 2>"$errfile"
|
|
1345
|
+
status=$?
|
|
1346
|
+
if [ "$status" -ne 0 ]; then
|
|
1347
|
+
IFS= read -r REDISPATCH_ERR <"$errfile" || :
|
|
1348
|
+
[ -n "$REDISPATCH_ERR" ] || REDISPATCH_ERR="dispatch exited $status"
|
|
1349
|
+
fi
|
|
1350
|
+
cat "$errfile" >&2
|
|
1351
|
+
rm -f "$errfile"
|
|
1352
|
+
return "$status"
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
# next_chain_var <status_json> — NEXT_CHAIN is the bundle's `chain` + 1.
|
|
1356
|
+
# 1: the chain is unreadable; 2: NEXT_CHAIN is over MAX_CHAIN.
|
|
1357
|
+
NEXT_CHAIN=""
|
|
1358
|
+
next_chain_var() {
|
|
1359
|
+
local current
|
|
1360
|
+
NEXT_CHAIN=""
|
|
1361
|
+
current=$(hr_remote_status_get "$1" chain) || return 1
|
|
1362
|
+
case "$current" in
|
|
1363
|
+
''|*[!0-9]*) return 1 ;;
|
|
1364
|
+
esac
|
|
1365
|
+
NEXT_CHAIN=$((10#$current + 1))
|
|
1366
|
+
[ "$NEXT_CHAIN" -le "$((10#$MAX_CHAIN))" ] || return 2
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
valid_engine() {
|
|
1370
|
+
case "${1-}" in
|
|
1371
|
+
task|user_review|docs) return 0 ;;
|
|
1372
|
+
esac
|
|
1373
|
+
return 1
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
STOPPED_LINE="a 'harness stop' run is newer than its newest 'harness run' run"
|
|
1377
|
+
RESUME_HINT="/autonomous-sdlc-harness:branch-resume"
|
|
1378
|
+
|
|
1379
|
+
continue_redispatch() {
|
|
1380
|
+
local status_file="$1" engine_value
|
|
1381
|
+
if [ -n "${HARNESS_REMOTE_STOP-}" ]; then
|
|
1382
|
+
notify paused "$branch" "Not re-dispatched: remote stop is set. Run $RESUME_HINT $branch to continue."
|
|
1383
|
+
return 0
|
|
1384
|
+
fi
|
|
1385
|
+
remote_branch_stopped "$branch"
|
|
1386
|
+
case $? in
|
|
1387
|
+
0) echo "remote-run.sh: $branch is stopped ($STOPPED_LINE); not re-dispatched"; return 0 ;;
|
|
1388
|
+
2) notify paused "$branch" "Not re-dispatched: the stop-marker check failed ($GH_ERR). Run $RESUME_HINT $branch to continue."; return 0 ;;
|
|
1389
|
+
esac
|
|
1390
|
+
if ! max_chain_var; then
|
|
1391
|
+
notify failed "$branch" "Not re-dispatched: HARNESS_MAX_CHAIN '$MAX_CHAIN' is not a non-negative integer."
|
|
1392
|
+
return 0
|
|
1393
|
+
fi
|
|
1394
|
+
next_chain_var "$status_file"
|
|
1395
|
+
case $? in
|
|
1396
|
+
1) notify failed "$branch" "Not re-dispatched: chain unreadable in status.json."; return 0 ;;
|
|
1397
|
+
2) notify failed "$branch" "Not re-dispatched: chain limit reached ($NEXT_CHAIN over HARNESS_MAX_CHAIN $MAX_CHAIN)."; return 0 ;;
|
|
1398
|
+
esac
|
|
1399
|
+
engine_value=$(hr_remote_status_get "$status_file" engine) || engine_value=""
|
|
1400
|
+
if ! valid_engine "$engine_value"; then
|
|
1401
|
+
notify failed "$branch" "Not re-dispatched: engine '$engine_value' in status.json is not task, user_review or docs."
|
|
1402
|
+
return 0
|
|
1403
|
+
fi
|
|
1404
|
+
redispatch "$engine_value" "$NEXT_CHAIN" \
|
|
1405
|
+
|| notify paused "$branch" "Re-dispatch failed ($REDISPATCH_ERR). Run $RESUME_HINT $branch to continue."
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
continue_wait_poller() {
|
|
1409
|
+
remote_branch_stopped "$branch"
|
|
1410
|
+
case $? in
|
|
1411
|
+
0) echo "remote-run.sh: $branch is stopped ($STOPPED_LINE); the resume poller is not enabled"; return 0 ;;
|
|
1412
|
+
2) notify paused "$branch" "Auto-resume not enabled: the stop-marker check failed ($GH_ERR). Run $RESUME_HINT $branch to continue."; return 0 ;;
|
|
1413
|
+
esac
|
|
1414
|
+
if gh_call workflow enable "$WORKFLOW_RESUME_FILE"; then
|
|
1415
|
+
echo "remote-run.sh: enabled $WORKFLOW_RESUME_FILE for $branch"
|
|
1416
|
+
else
|
|
1417
|
+
notify paused "$branch" "Auto-resume is unavailable: enabling $WORKFLOW_RESUME_FILE failed ($GH_ERR). Run $RESUME_HINT $branch after the usage reset."
|
|
1418
|
+
fi
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
verb_continue() {
|
|
1422
|
+
local status_file decision
|
|
1423
|
+
hr_remote_names_var
|
|
1424
|
+
status_file="$bundle_dir/$HR_REMOTE_STATUS_FILE"
|
|
1425
|
+
if [ ! -f "$status_file" ]; then
|
|
1426
|
+
notify failed "$branch" "The job stopped before the harness run started: $(this_run_url)"
|
|
1427
|
+
return 0
|
|
1428
|
+
fi
|
|
1429
|
+
decision=$(hr_remote_status_get "$status_file" decision) || decision=""
|
|
1430
|
+
case "$decision" in
|
|
1431
|
+
continue) continue_redispatch "$status_file" ;;
|
|
1432
|
+
wait-poller) continue_wait_poller ;;
|
|
1433
|
+
stop) echo "remote-run.sh: decision stop for $branch; nothing to do" ;;
|
|
1434
|
+
*) notify failed "$branch" "Not re-dispatched: status.json carries no recognised decision: $(this_run_url)" ;;
|
|
1435
|
+
esac
|
|
1436
|
+
return 0
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
# POLL_STATE — the poller state carried between ticks, one object keyed by
|
|
1440
|
+
# branch: {"<branch>": {"run_id", "failures", "notified"}}, every value a string.
|
|
1441
|
+
POLL_STATE='{}'
|
|
1442
|
+
POLL_STATE_FILE_NAME='poll_state.json'
|
|
1443
|
+
|
|
1444
|
+
poll_state_get() {
|
|
1445
|
+
printf '%s' "$POLL_STATE" | jq -r --arg b "$1" --arg f "$2" '.[$b][$f] // "" | tostring'
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
poll_state_put() {
|
|
1449
|
+
POLL_STATE=$(printf '%s' "$POLL_STATE" | jq -c --arg b "$1" --arg r "$2" --arg f "$3" --arg n "$4" \
|
|
1450
|
+
'.[$b] = {run_id: $r, failures: $f, notified: $n}')
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
poll_state_drop() {
|
|
1454
|
+
POLL_STATE=$(printf '%s' "$POLL_STATE" | jq -c --arg b "$1" 'del(.[$b])')
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
# poll_state_load — POLL_STATE from the newest other run of the poller carrying
|
|
1458
|
+
# the poll-state artifact. Any failure is one line and an empty state.
|
|
1459
|
+
poll_state_load() {
|
|
1460
|
+
local previous ids id found="" file parsed
|
|
1461
|
+
POLL_STATE='{}'
|
|
1462
|
+
previous=$(hr_state_path "$root" autonomous_logs/poll_state/previous) || {
|
|
1463
|
+
echo "remote-run.sh: poll: cannot resolve the poller state directory; starting from an empty state"
|
|
1464
|
+
return 0
|
|
1465
|
+
}
|
|
1466
|
+
if ! gh_call run list --workflow "$WORKFLOW_RESUME_FILE" --json databaseId,createdAt --limit "$POLL_STATE_RUNS_LIMIT"; then
|
|
1467
|
+
echo "remote-run.sh: poll: listing the runs of $WORKFLOW_RESUME_FILE failed ($GH_ERR); starting from an empty state"
|
|
1468
|
+
return 0
|
|
1469
|
+
fi
|
|
1470
|
+
ids=$(printf '%s' "$GH_OUT" | jq -r --arg self "${GITHUB_RUN_ID-}" '
|
|
1471
|
+
[.[] | select((.databaseId | tostring) != $self)]
|
|
1472
|
+
| sort_by([.createdAt, .databaseId]) | reverse | .[].databaseId | tostring' 2>/dev/null) || {
|
|
1473
|
+
echo "remote-run.sh: poll: the run list of $WORKFLOW_RESUME_FILE is not the expected JSON; starting from an empty state"
|
|
1474
|
+
return 0
|
|
1475
|
+
}
|
|
1476
|
+
for id in $ids; do
|
|
1477
|
+
bundle_listed "$id" "$POLL_STATE_ARTIFACT_NAME"
|
|
1478
|
+
case $? in
|
|
1479
|
+
0) found="$id"; break ;;
|
|
1480
|
+
2) echo "remote-run.sh: poll: reading the artifacts of poller run $id failed ($GH_ERR); starting from an empty state"; return 0 ;;
|
|
1481
|
+
esac
|
|
1482
|
+
done
|
|
1483
|
+
if [ -z "$found" ]; then
|
|
1484
|
+
echo "remote-run.sh: poll: no earlier poller run carries $POLL_STATE_ARTIFACT_NAME; starting from an empty state"
|
|
1485
|
+
return 0
|
|
1486
|
+
fi
|
|
1487
|
+
if ! mkdir -p "$previous" || ! rm -f "$previous/$POLL_STATE_FILE_NAME"; then
|
|
1488
|
+
echo "remote-run.sh: poll: cannot create '$previous'; starting from an empty state"
|
|
1489
|
+
return 0
|
|
1490
|
+
fi
|
|
1491
|
+
if ! gh_call run download "$found" -n "$POLL_STATE_ARTIFACT_NAME" -D "$previous"; then
|
|
1492
|
+
echo "remote-run.sh: poll: downloading the poller state of run $found failed ($GH_ERR); starting from an empty state"
|
|
1493
|
+
return 0
|
|
1494
|
+
fi
|
|
1495
|
+
file="$previous/$POLL_STATE_FILE_NAME"
|
|
1496
|
+
parsed=$(jq -c 'if type == "object" then with_entries(select(.value | type == "object")) else error("not an object") end' \
|
|
1497
|
+
"$file" 2>/dev/null)
|
|
1498
|
+
if [ -z "$parsed" ]; then
|
|
1499
|
+
echo "remote-run.sh: poll: '$file' is not a poller state object; starting from an empty state"
|
|
1500
|
+
return 0
|
|
1501
|
+
fi
|
|
1502
|
+
POLL_STATE="$parsed"
|
|
1503
|
+
echo "remote-run.sh: poll: carried the poller state of run $found"
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
# poll_state_write — POLL_STATE into `current/`, which the poller uploads.
|
|
1507
|
+
poll_state_write() {
|
|
1508
|
+
local current
|
|
1509
|
+
current=$(hr_state_path "$root" autonomous_logs/poll_state/current) \
|
|
1510
|
+
&& mkdir -p "$current" \
|
|
1511
|
+
&& printf '%s\n' "$POLL_STATE" >"$current/$POLL_STATE_FILE_NAME" \
|
|
1512
|
+
|| echo "remote-run.sh: poll: writing the poller state failed; the next tick starts from an empty state" >&2
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
# poll_bounds_var — POLL_MAX_FAILURES and POLL_GIVE_UP_MINUTES from the
|
|
1516
|
+
# environment. 1 with POLL_BOUND_BAD naming the value that is not a
|
|
1517
|
+
# non-negative integer.
|
|
1518
|
+
POLL_MAX_FAILURES=""
|
|
1519
|
+
POLL_GIVE_UP_MINUTES=""
|
|
1520
|
+
POLL_BOUND_BAD=""
|
|
1521
|
+
poll_bounds_var() {
|
|
1522
|
+
POLL_MAX_FAILURES="${HARNESS_POLL_MAX_DISPATCH_FAILURES:-$POLL_MAX_DISPATCH_FAILURES_DEFAULT}"
|
|
1523
|
+
POLL_GIVE_UP_MINUTES="${HARNESS_POLL_GIVE_UP_AFTER_MINUTES:-$POLL_GIVE_UP_AFTER_MINUTES_DEFAULT}"
|
|
1524
|
+
case "$POLL_MAX_FAILURES" in
|
|
1525
|
+
*[!0-9]*) POLL_BOUND_BAD="HARNESS_POLL_MAX_DISPATCH_FAILURES '$POLL_MAX_FAILURES'"; return 1 ;;
|
|
1526
|
+
esac
|
|
1527
|
+
case "$POLL_GIVE_UP_MINUTES" in
|
|
1528
|
+
*[!0-9]*) POLL_BOUND_BAD="HARNESS_POLL_GIVE_UP_AFTER_MINUTES '$POLL_GIVE_UP_MINUTES'"; return 1 ;;
|
|
1529
|
+
esac
|
|
1530
|
+
return 0
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
# poll_fetch <run_id> — POLL_STATUS_FILE is the status.json of the run's
|
|
1534
|
+
# bundle, downloaded unless its directory already holds it. 1 on failure, with
|
|
1535
|
+
# one line said.
|
|
1536
|
+
POLL_STATUS_FILE=""
|
|
1537
|
+
poll_fetch() {
|
|
1538
|
+
local id="$1" download
|
|
1539
|
+
POLL_STATUS_FILE=""
|
|
1540
|
+
download=$(hr_state_path "$root" "autonomous_logs/remote_download/$branch/$id") || {
|
|
1541
|
+
echo "remote-run.sh: poll: cannot resolve '$root/harness.config.json' for $branch" >&2
|
|
1542
|
+
return 1
|
|
1543
|
+
}
|
|
1544
|
+
POLL_STATUS_FILE="$download/$HR_REMOTE_STATUS_FILE"
|
|
1545
|
+
[ ! -f "$POLL_STATUS_FILE" ] || return 0
|
|
1546
|
+
if ! mkdir -p "$download"; then
|
|
1547
|
+
echo "remote-run.sh: poll: cannot create '$download' for $branch" >&2
|
|
1548
|
+
return 1
|
|
1549
|
+
fi
|
|
1550
|
+
if ! gh_call run download "$id" -n "$STATE_ARTIFACT_NAME" -D "$download"; then
|
|
1551
|
+
echo "remote-run.sh: poll: the bundle of run $id ($branch) cannot be downloaded: $GH_ERR"
|
|
1552
|
+
return 1
|
|
1553
|
+
fi
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
# poll_usage_paused — 0 when POLL_STATUS_FILE says paused / usage.
|
|
1557
|
+
poll_usage_paused() {
|
|
1558
|
+
local status reason
|
|
1559
|
+
status=$(hr_remote_status_get "$POLL_STATUS_FILE" status) || status=""
|
|
1560
|
+
reason=$(hr_remote_status_get "$POLL_STATUS_FILE" pause_reason) || reason=""
|
|
1561
|
+
[ "$status" = paused ] && [ "$reason" = usage ]
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
# poll_branch <run_id> <state> <may_dispatch> — one branch's newest `harness
|
|
1565
|
+
# run` run; the global branch names it. Returns 0 when the branch is waiting.
|
|
1566
|
+
# With may_dispatch 0 nothing is sent and nothing notified: a due run that
|
|
1567
|
+
# would be dispatched counts as waiting, one that would be refused does not.
|
|
1568
|
+
poll_branch() {
|
|
1569
|
+
local id="$1" state="$2" may_dispatch="$3" at engine_value now failures
|
|
1570
|
+
if [ "$may_dispatch" -eq 1 ] && [ -n "$(poll_state_get "$branch" run_id)" ] \
|
|
1571
|
+
&& [ "$(poll_state_get "$branch" run_id)" != "$id" ]; then
|
|
1572
|
+
poll_state_drop "$branch"
|
|
1573
|
+
fi
|
|
1574
|
+
remote_branch_stopped "$branch"
|
|
1575
|
+
case $? in
|
|
1576
|
+
0) echo "remote-run.sh: poll: $branch is stopped ($STOPPED_LINE); skipped"; return 1 ;;
|
|
1577
|
+
2) echo "remote-run.sh: poll: the stop-marker check for $branch failed ($GH_ERR); skipped"; return 1 ;;
|
|
1578
|
+
esac
|
|
1579
|
+
if [ "$state" != completed ]; then
|
|
1580
|
+
# Never dispatched from here: the job's own `continue` step enables the
|
|
1581
|
+
# poller after its upload, and a later tick sees the run completed.
|
|
1582
|
+
bundle_listed "$id"
|
|
1583
|
+
case $? in
|
|
1584
|
+
1) return 1 ;;
|
|
1585
|
+
2) echo "remote-run.sh: poll: reading the artifacts of unfinished run $id ($branch) failed ($GH_ERR); counted as waiting"; return 0 ;;
|
|
1586
|
+
esac
|
|
1587
|
+
if ! poll_fetch "$id"; then
|
|
1588
|
+
echo "remote-run.sh: poll: unfinished run $id ($branch) counted as waiting"
|
|
1589
|
+
return 0
|
|
1590
|
+
fi
|
|
1591
|
+
poll_usage_paused || return 1
|
|
1592
|
+
echo "remote-run.sh: poll: $branch's unfinished run $id carries a usage-paused bundle; waiting"
|
|
1593
|
+
return 0
|
|
1594
|
+
fi
|
|
1595
|
+
if [ "$(poll_state_get "$branch" run_id)" = "$id" ] && [ "$(poll_state_get "$branch" notified)" = 1 ]; then
|
|
1596
|
+
echo "remote-run.sh: poll: $branch's run $id was already reported as not re-dispatchable; skipped"
|
|
1597
|
+
return 1
|
|
1598
|
+
fi
|
|
1599
|
+
if ! poll_fetch "$id"; then
|
|
1600
|
+
echo "remote-run.sh: poll: $branch skipped"
|
|
1601
|
+
return 1
|
|
1602
|
+
fi
|
|
1603
|
+
poll_usage_paused || return 1
|
|
1604
|
+
at=$(hr_remote_status_get "$POLL_STATUS_FILE" usage_resume_at) || at=""
|
|
1605
|
+
case "$at" in
|
|
1606
|
+
''|*[!0-9]*) echo "remote-run.sh: poll: $branch is usage-paused with no readable usage_resume_at; skipped"; return 1 ;;
|
|
1607
|
+
esac
|
|
1608
|
+
now=$(date +%s)
|
|
1609
|
+
if [ "$((10#$at))" -gt "$now" ]; then
|
|
1610
|
+
echo "remote-run.sh: poll: $branch waits for its usage reset at $at"
|
|
1611
|
+
return 0
|
|
1612
|
+
fi
|
|
1613
|
+
next_chain_var "$POLL_STATUS_FILE"
|
|
1614
|
+
case $? in
|
|
1615
|
+
1) [ "$may_dispatch" -eq 0 ] || notify failed "$branch" "Not resumed by the poller: chain unreadable in status.json."; return 1 ;;
|
|
1616
|
+
2) [ "$may_dispatch" -eq 0 ] || notify failed "$branch" "Not resumed by the poller: chain limit reached ($NEXT_CHAIN over HARNESS_MAX_CHAIN $MAX_CHAIN)."; return 1 ;;
|
|
1617
|
+
esac
|
|
1618
|
+
engine_value=$(hr_remote_status_get "$POLL_STATUS_FILE" engine) || engine_value=""
|
|
1619
|
+
if ! valid_engine "$engine_value"; then
|
|
1620
|
+
[ "$may_dispatch" -eq 0 ] || notify failed "$branch" "Not resumed by the poller: engine '$engine_value' in status.json is not task, user_review or docs."
|
|
1621
|
+
return 1
|
|
1622
|
+
fi
|
|
1623
|
+
if [ "$may_dispatch" -eq 0 ]; then
|
|
1624
|
+
echo "remote-run.sh: poll: $branch is due; the next tick dispatches it"
|
|
1625
|
+
return 0
|
|
1626
|
+
fi
|
|
1627
|
+
if redispatch "$engine_value" "$NEXT_CHAIN"; then
|
|
1628
|
+
echo "remote-run.sh: poll: dispatched $branch --resume pause --chain $NEXT_CHAIN"
|
|
1629
|
+
POLL_DISPATCHED="$POLL_DISPATCHED$branch "
|
|
1630
|
+
poll_state_drop "$branch"
|
|
1631
|
+
return 1
|
|
1632
|
+
fi
|
|
1633
|
+
failures=$(poll_state_get "$branch" failures)
|
|
1634
|
+
case "$failures" in
|
|
1635
|
+
''|*[!0-9]*) failures=0 ;;
|
|
1636
|
+
esac
|
|
1637
|
+
failures=$((10#$failures + 1))
|
|
1638
|
+
if [ "$failures" -ge "$((10#$POLL_MAX_FAILURES))" ] \
|
|
1639
|
+
|| [ "$now" -gt "$((10#$at + 10#$POLL_GIVE_UP_MINUTES * 60))" ]; then
|
|
1640
|
+
poll_state_put "$branch" "$id" "$failures" 1
|
|
1641
|
+
notify paused "$branch" "The resume poller could not re-dispatch $branch ($REDISPATCH_ERR) after $failures attempts; automatic resume has stopped. Run $RESUME_HINT $branch."
|
|
1642
|
+
return 1
|
|
1643
|
+
fi
|
|
1644
|
+
poll_state_put "$branch" "$id" "$failures" ""
|
|
1645
|
+
echo "remote-run.sh: poll: dispatching $branch failed ($REDISPATCH_ERR), attempt $failures of $POLL_MAX_FAILURES; still waiting"
|
|
1646
|
+
return 0
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
# poll_pass <may_dispatch> — poll_branch over each branch's newest `harness run`
|
|
1650
|
+
# run in ALL_RUNS, skipping a branch this tick already dispatched. POLL_WAITING
|
|
1651
|
+
# lists the waiting branches, space-separated, and POLL_WAITING_COUNT counts
|
|
1652
|
+
# them. 1 when ALL_RUNS is unreadable.
|
|
1653
|
+
POLL_WAITING=""
|
|
1654
|
+
POLL_WAITING_COUNT=0
|
|
1655
|
+
POLL_DISPATCHED=" "
|
|
1656
|
+
poll_pass() {
|
|
1657
|
+
local entries b id state
|
|
1658
|
+
POLL_WAITING=""
|
|
1659
|
+
POLL_WAITING_COUNT=0
|
|
1660
|
+
entries=$(printf '%s' "$ALL_RUNS" | jq -r '
|
|
1661
|
+
[.[] | select((.displayTitle // "") | startswith("harness run "))]
|
|
1662
|
+
| group_by(.displayTitle)
|
|
1663
|
+
| map(sort_by([.createdAt, .databaseId]) | last)
|
|
1664
|
+
| .[] | [(.displayTitle | ltrimstr("harness run ")), (.databaseId | tostring), (.status // "")] | @tsv' 2>/dev/null) || {
|
|
1665
|
+
GH_ERR="its run list is not the expected JSON"
|
|
1666
|
+
return 1
|
|
1667
|
+
}
|
|
1668
|
+
while IFS=$'\t' read -r b id state; do
|
|
1669
|
+
valid_branch "$b" || continue
|
|
1670
|
+
case "$POLL_DISPATCHED" in *" $b "*) continue ;; esac
|
|
1671
|
+
branch="$b"
|
|
1672
|
+
if poll_branch "$id" "$state" "$1"; then
|
|
1673
|
+
POLL_WAITING="$POLL_WAITING$b "
|
|
1674
|
+
POLL_WAITING_COUNT=$((POLL_WAITING_COUNT + 1))
|
|
1675
|
+
fi
|
|
1676
|
+
done <<EOF
|
|
1677
|
+
$entries
|
|
1678
|
+
EOF
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
# poll_recheck — after the disable: one fresh listing, evaluated without
|
|
1682
|
+
# dispatching; re-enables the poller when a branch became waiting meanwhile.
|
|
1683
|
+
poll_recheck() {
|
|
1684
|
+
local b
|
|
1685
|
+
ALL_RUNS_LISTED=0
|
|
1686
|
+
if ! list_all_runs || ! poll_pass 0; then
|
|
1687
|
+
echo "remote-run.sh: poll: the re-check after disabling $WORKFLOW_RESUME_FILE could not list the runs ($GH_ERR); it stays disabled"
|
|
1688
|
+
return 0
|
|
1689
|
+
fi
|
|
1690
|
+
[ "$POLL_WAITING_COUNT" -gt 0 ] || return 0
|
|
1691
|
+
if gh_call workflow enable "$WORKFLOW_RESUME_FILE"; then
|
|
1692
|
+
for b in $POLL_WAITING; do
|
|
1693
|
+
echo "remote-run.sh: poll: $b became waiting during this tick; re-enabled $WORKFLOW_RESUME_FILE"
|
|
1694
|
+
done
|
|
1695
|
+
return 0
|
|
1696
|
+
fi
|
|
1697
|
+
for b in $POLL_WAITING; do
|
|
1698
|
+
notify paused "$b" "Auto-resume is unavailable: re-enabling $WORKFLOW_RESUME_FILE failed ($GH_ERR). Run $RESUME_HINT $b after the usage reset."
|
|
1699
|
+
done
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
# verb_poll writes the poller state on every exit, so the next tick inherits
|
|
1703
|
+
# the count whatever this one met.
|
|
1704
|
+
verb_poll() {
|
|
1705
|
+
poll_state_load
|
|
1706
|
+
if [ -n "${HARNESS_REMOTE_STOP-}" ]; then
|
|
1707
|
+
poll_state_write
|
|
1708
|
+
echo "remote-run.sh: poll: remote stop is set; nothing dispatched"
|
|
1709
|
+
return 0
|
|
1710
|
+
fi
|
|
1711
|
+
if ! max_chain_var; then
|
|
1712
|
+
poll_state_write
|
|
1713
|
+
echo "remote-run.sh: poll: HARNESS_MAX_CHAIN '$MAX_CHAIN' is not a non-negative integer; nothing dispatched" >&2
|
|
1714
|
+
exit "$EXIT_USAGE"
|
|
1715
|
+
fi
|
|
1716
|
+
if ! poll_bounds_var; then
|
|
1717
|
+
poll_state_write
|
|
1718
|
+
echo "remote-run.sh: poll: $POLL_BOUND_BAD is not a non-negative integer; nothing dispatched" >&2
|
|
1719
|
+
exit "$EXIT_USAGE"
|
|
1720
|
+
fi
|
|
1721
|
+
hr_remote_names_var
|
|
1722
|
+
if ! list_all_runs || ! poll_pass 1; then
|
|
1723
|
+
poll_state_write
|
|
1724
|
+
gh_fail "listing the runs of $WORKFLOW_RUN_FILE failed"
|
|
1725
|
+
fi
|
|
1726
|
+
poll_state_write
|
|
1727
|
+
if [ "$POLL_WAITING_COUNT" -gt 0 ]; then
|
|
1728
|
+
echo "remote-run.sh: poll: $POLL_WAITING_COUNT branch(es) still waiting; $WORKFLOW_RESUME_FILE stays enabled"
|
|
1729
|
+
return 0
|
|
1730
|
+
fi
|
|
1731
|
+
gh_call workflow disable "$WORKFLOW_RESUME_FILE" || gh_fail "disabling $WORKFLOW_RESUME_FILE failed"
|
|
1732
|
+
echo "remote-run.sh: poll: no branch is waiting; disabled $WORKFLOW_RESUME_FILE"
|
|
1733
|
+
poll_recheck
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
verb_pause_requested() {
|
|
1737
|
+
local found
|
|
1738
|
+
list_runs
|
|
1739
|
+
# An unparseable createdAt is no match rather than a failed read.
|
|
1740
|
+
found=$(printf '%s' "$GH_OUT" | jq -r --arg t "harness pause $branch" --argjson since "$since_arg" '
|
|
1741
|
+
[.[] | select(.displayTitle == $t)
|
|
1742
|
+
| ((.createdAt // "") | try fromdateiso8601 catch null)
|
|
1743
|
+
| select(. != null and . >= $since)] | length' 2>/dev/null)
|
|
1744
|
+
case "$found" in
|
|
1745
|
+
''|*[!0-9]*)
|
|
1746
|
+
GH_ERR="its run list is not the expected JSON"
|
|
1747
|
+
gh_fail "listing the runs of '$branch' failed"
|
|
1748
|
+
;;
|
|
1749
|
+
esac
|
|
1750
|
+
if [ "$found" -gt 0 ]; then
|
|
1751
|
+
echo "remote-run.sh: a 'harness pause $branch' run was created at or after $since_arg"
|
|
1752
|
+
return 0
|
|
1753
|
+
fi
|
|
1754
|
+
echo "remote-run.sh: no 'harness pause $branch' run was created at or after $since_arg"
|
|
1755
|
+
exit "$EXIT_NO_PAUSE"
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
verb_run_created_at() {
|
|
1759
|
+
local epoch
|
|
1760
|
+
gh_call run view "$run_id_arg" --json createdAt || gh_fail "reading run $run_id_arg failed"
|
|
1761
|
+
epoch=$(printf '%s' "$GH_OUT" | jq -r '.createdAt | fromdateiso8601 | floor' 2>/dev/null)
|
|
1762
|
+
case "$epoch" in
|
|
1763
|
+
''|*[!0-9]*)
|
|
1764
|
+
GH_ERR="its createdAt is not an ISO 8601 UTC time"
|
|
1765
|
+
gh_fail "reading run $run_id_arg failed"
|
|
1766
|
+
;;
|
|
1767
|
+
esac
|
|
1768
|
+
printf '%s\n' "$epoch"
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
case "$verb" in
|
|
1772
|
+
dispatch) verb_dispatch ;;
|
|
1773
|
+
pause) verb_pause ;;
|
|
1774
|
+
warm) verb_warm ;;
|
|
1775
|
+
stop) verb_stop ;;
|
|
1776
|
+
status) verb_status ;;
|
|
1777
|
+
sync) verb_sync ;;
|
|
1778
|
+
restore) verb_restore ;;
|
|
1779
|
+
save) verb_save ;;
|
|
1780
|
+
continue) verb_continue ;;
|
|
1781
|
+
poll) verb_poll ;;
|
|
1782
|
+
pause-requested) verb_pause_requested ;;
|
|
1783
|
+
run-created-at) verb_run_created_at ;;
|
|
1784
|
+
esac
|
|
1785
|
+
exit "$EXIT_OK"
|