@indigoai-us/hq-cli 5.345.117 → 5.345.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/assets/scaffold/core/scripts/pipeline-conductor.sh +2671 -0
- package/assets/scaffold/core/scripts/pipeline-driver.sh +820 -0
- package/assets/scaffold/core/scripts/pipeline-envelope.sh +211 -0
- package/dist/command-catalog.generated.d.ts +34 -0
- package/dist/command-catalog.generated.js +40 -0
- package/dist/commands/core-checkpoint-native.d.ts +17 -3
- package/dist/commands/core-checkpoint-native.js +48 -8
- package/dist/commands/core-derive-trigger-facts-native.js +15 -7
- package/dist/commands/core-migrate-policy-triggers-native.js +3 -1
- package/dist/commands/core-pipeline-conductor-command.d.ts +5 -0
- package/dist/commands/core-pipeline-conductor-command.js +138 -0
- package/dist/commands/core-pipeline-conductor-native.d.ts +7 -0
- package/dist/commands/core-pipeline-conductor-native.js +2950 -0
- package/dist/commands/core-pipeline-driver-native.d.ts +24 -0
- package/dist/commands/core-pipeline-driver-native.js +843 -0
- package/dist/commands/core-pipeline-driver-process.d.ts +4 -0
- package/dist/commands/core-pipeline-driver-process.js +68 -0
- package/dist/commands/core-pipeline-envelope-native.d.ts +4 -0
- package/dist/commands/core-pipeline-envelope-native.js +87 -0
- package/dist/commands/core-policy.js +3 -0
- package/dist/commands/core-worker-lint-native.d.ts +0 -1
- package/dist/commands/core-worker-lint-native.js +0 -1
- package/dist/commands/core-worker-share-native.js +2 -2
- package/dist/commands/core-worker-skills-command.d.ts +1 -1
- package/dist/commands/core-worker-skills-command.js +3 -20
- package/dist/commands/core.d.ts +5 -1
- package/dist/commands/core.js +8 -9
- package/dist/commands/integrations-core.d.ts +7 -1
- package/dist/commands/integrations-core.js +17 -4
- package/dist/commands/integrations.js +91 -22
- package/dist/commands/storage-gate.d.ts +4 -3
- package/dist/commands/storage-gate.js +4 -3
- package/dist/fast-scaffold.js +0 -2
- package/dist/lib/storage-feature-flag.d.ts +7 -7
- package/package.json +1 -1
- package/assets/scaffold/core/scripts/lint-shared-worker-skills.sh +0 -197
- package/assets/scaffold/core/scripts/share-worker-skill.sh +0 -178
- package/dist/commands/core-native-worker-skills-flag.d.ts +0 -4
- package/dist/commands/core-native-worker-skills-flag.js +0 -7
|
@@ -0,0 +1,820 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# hq-core: public
|
|
3
|
+
# pipeline-driver.sh - the detached loop behind /run-project --pipeline.
|
|
4
|
+
#
|
|
5
|
+
# A lane cannot keep a child alive past its own engine call, so the loop that
|
|
6
|
+
# moves a pipeline forward does not live in a lane. The parent session starts
|
|
7
|
+
# this script through hq-detach.sh and arms a waiter on it. It makes no engine
|
|
8
|
+
# calls. Each pass it:
|
|
9
|
+
# 1. moves a stale handoff of a re-queued phase aside (handoffs/<id>-<phase>.failed.<n>.json),
|
|
10
|
+
# 2. accepts the handoff a loop lane wrote to result_path for any in-flight
|
|
11
|
+
# phase (pipeline-conductor.sh accept),
|
|
12
|
+
# 3. runs recheck for every story awaiting it,
|
|
13
|
+
# 4. ticks the conductor to route the next phase or start the next story,
|
|
14
|
+
# and exits when the parent is needed or the run is over.
|
|
15
|
+
#
|
|
16
|
+
# A handoff with status `failed` is routed again until the phase has failed
|
|
17
|
+
# --max-phase-fails times. A handoff with status `blocked` is never routed
|
|
18
|
+
# again: the conductor holds that story as blocked_needs_owner and writes a
|
|
19
|
+
# decision item, and the driver keeps routing every story that does not depend
|
|
20
|
+
# on it. It exits 21 for the blocked story only when nothing else can move.
|
|
21
|
+
#
|
|
22
|
+
# Usage:
|
|
23
|
+
# pipeline-driver.sh --prd <prd.json> --state <dir>
|
|
24
|
+
# [--worktree <dir> | --worktree <repo>=<dir> ...]
|
|
25
|
+
# [--story-branches] [--table <table.tsv>] [--interval <seconds>]
|
|
26
|
+
# [--max-phase-fails <n>] [--stall-window <seconds>]
|
|
27
|
+
# [--allow-repos-worktree]
|
|
28
|
+
#
|
|
29
|
+
# --allow-repos-worktree passes through to tick: a --worktree under repos/ is
|
|
30
|
+
# otherwise refused (pipeline-conductor.sh explains why), and with it every
|
|
31
|
+
# envelope routed there tells the worker to edit through the shell or apply_patch.
|
|
32
|
+
#
|
|
33
|
+
# --worktree is repeatable in the <repo>=<dir> form: each story's phases go to
|
|
34
|
+
# the worktree of the story's repo (pipeline-conductor.sh documents how the repo
|
|
35
|
+
# is resolved). The bare form serves a prd with one repo. --story-branches
|
|
36
|
+
# passes through to tick (one stacked branch per story instead of one feature
|
|
37
|
+
# branch per repo). --table is the confirmed worker table (pipeline/table.tsv);
|
|
38
|
+
# it passes through to tick, which records it in run.json, and the conductor
|
|
39
|
+
# then refuses to route a phase whose worker has no row in it (exit 27 below).
|
|
40
|
+
#
|
|
41
|
+
# Launch (from the HQ root):
|
|
42
|
+
# bash core/scripts/hq-detach.sh --logfile <state>/driver/detach.log -- \
|
|
43
|
+
# sh core/scripts/pipeline-driver.sh --prd <prd.json> --state <state> --worktree <dir>
|
|
44
|
+
#
|
|
45
|
+
# Files under <state>/driver/:
|
|
46
|
+
# driver.pid this driver's pid while it runs
|
|
47
|
+
# stop.json written by `pipeline-conductor.sh stop` (a stop envelope,
|
|
48
|
+
# {"kind":"stop",...}); the driver reads it each pass and stops
|
|
49
|
+
# lock/ held while it runs; a second copy against the same state dir
|
|
50
|
+
# refuses to start (exit 24) unless the holder is dead
|
|
51
|
+
# driver.log append-only, one timestamped line per action
|
|
52
|
+
# exit "<code> <reason>" written on exit, for the parent's waiter
|
|
53
|
+
#
|
|
54
|
+
# Exit codes:
|
|
55
|
+
# code meaning the parent then
|
|
56
|
+
# 0 every story finished (verified, failed_report, reads report.md FINAL
|
|
57
|
+
# accepted_partial or parked); report.md has FINAL
|
|
58
|
+
# 2 usage error fixes the command
|
|
59
|
+
# 20 a regression gate is due runs it, records `gate
|
|
60
|
+
# result`, restarts
|
|
61
|
+
# 21 a decision is needed: a story is held for an reads the decision item,
|
|
62
|
+
# approval; a phase returned blocked, a story has asks the owner once, runs
|
|
63
|
+
# no worktree, or a story is awaiting_go, and the matching command,
|
|
64
|
+
# nothing else can move; the regression gate failed; or
|
|
65
|
+
# a phase returned failed --max-phase-fails times restarts the driver
|
|
66
|
+
# (the story is then held as blocked_needs_owner
|
|
67
|
+
# with decisions/<id>-blocked-<phase>.md)
|
|
68
|
+
# 22 an in-flight phase passed its envelope deadline checks the lane
|
|
69
|
+
# 23 the conductor printed something unexpected, or reads driver.log
|
|
70
|
+
# a conductor call failed
|
|
71
|
+
# 24 another driver already runs against this state leaves it running
|
|
72
|
+
# 25 retired: the stall check now fires the stall -
|
|
73
|
+
# event and exits 28
|
|
74
|
+
# 26 lane down: enqueue found a terminal loop lane; relaunches the named
|
|
75
|
+
# the mapping was dropped and the story stays worker's loop lane,
|
|
76
|
+
# queued; the reason names the lane then restarts the driver
|
|
77
|
+
# 27 no lane: a phase's worker has no row in the adds the row, launches
|
|
78
|
+
# confirmed worker table (--table); the story that lane, restarts the
|
|
79
|
+
# stays queued; the reason names the worker driver
|
|
80
|
+
# 28 stall event: for the whole --stall-window every reads events.jsonl, checks
|
|
81
|
+
# started lane is waiting with an empty queue and the named lanes, restarts
|
|
82
|
+
# a story is in flight, or an in-flight phase's the driver (the same stall
|
|
83
|
+
# envelope sits in its lane's queue with nothing does not fire again)
|
|
84
|
+
# active. <state>/events.jsonl gets
|
|
85
|
+
# {"event":"stall","stories":[...],"lanes":[...],"since":<iso>}
|
|
86
|
+
# 29 stopped on request (driver/stop.json from restarts the driver
|
|
87
|
+
# `pipeline-conductor.sh stop`) when it wants to resume
|
|
88
|
+
# 130/143 interrupted / terminated restarts the driver
|
|
89
|
+
#
|
|
90
|
+
# Interrupted phases: on every stop (stop.json, SIGTERM, SIGINT) the driver runs
|
|
91
|
+
# `pipeline-conductor.sh interrupt`: each in_flight story is marked `interrupted`
|
|
92
|
+
# with the phase and time in stories/<id>.json, and its envelope is withdrawn
|
|
93
|
+
# from the lane queue when no lane picked it up (a partial handoff is kept as
|
|
94
|
+
# handoffs/<id>-<phase>.interrupted.<n>.json). A loop lane that exited on a stop
|
|
95
|
+
# envelope while a story was in flight on it (journal `loop-done` after the
|
|
96
|
+
# route, no handoff) marks that story the same way. The next driver start routes
|
|
97
|
+
# interrupted stories first, before reopened ones, at the interrupted phase; the
|
|
98
|
+
# envelope carries resumed_after_interrupt: true and prior_handoff when a
|
|
99
|
+
# partial handoff exists. TICK lines and report.md FINAL name interrupted
|
|
100
|
+
# stories until they are routed again.
|
|
101
|
+
#
|
|
102
|
+
# Early engine exit: when a loop lane's engine call for a phase returns without a
|
|
103
|
+
# usable handoff, the lane writes a failed handoff with exit_reason
|
|
104
|
+
# "engine_exited_early" and a `phase-exit` event (reason, elapsed seconds) to its
|
|
105
|
+
# journal.jsonl. A handoff whose status is not passed, failed or blocked is not
|
|
106
|
+
# accepted; with a phase-exit event in the lane's journal since the phase was
|
|
107
|
+
# routed it counts the same way. On the next tick the driver logs EARLY_EXIT with
|
|
108
|
+
# the story, phase and elapsed time and accepts it as a failed phase, which routes
|
|
109
|
+
# the phase once more to the same lane (the one restart in place). A second early
|
|
110
|
+
# exit of the same phase holds the story for the owner (exit 21, the reason names
|
|
111
|
+
# engine_exited_early). While the engine is still running the deadline path (22)
|
|
112
|
+
# is unchanged.
|
|
113
|
+
#
|
|
114
|
+
# Reopen: when a regression gate fails because of a story that is already
|
|
115
|
+
# verified, the parent records `pipeline-conductor.sh gate result fail --story
|
|
116
|
+
# <id> --note <text>` (or runs `reopen`). The conductor queues that story again
|
|
117
|
+
# at its first implementing phase with the note in the envelope, and the gate
|
|
118
|
+
# turns `reopened`, which does not stop routing. A restarted driver routes the
|
|
119
|
+
# reopened story before any other queued story; once it verifies again the gate
|
|
120
|
+
# is due (exit 20). Verified dependents are not reopened: report.md FINAL lists
|
|
121
|
+
# them as "verified before <id> was reopened".
|
|
122
|
+
#
|
|
123
|
+
# Awaiting go: a story that needs an explicit go (approval: "explicit", or the
|
|
124
|
+
# conductor's release detector) is held awaiting_go before its first
|
|
125
|
+
# implementing phase. The driver keeps routing the rest, appends the list of
|
|
126
|
+
# awaiting_go stories to every TICK line in driver.log, and names them in the
|
|
127
|
+
# exit 21 reason and in report.md FINAL. `pipeline-conductor.sh go --story <id>`
|
|
128
|
+
# releases one; restart the driver after it.
|
|
129
|
+
#
|
|
130
|
+
# Restart safety: every decision is read from the state dir, never from memory.
|
|
131
|
+
# The conductor only accepts a handoff for a story that is in_flight on that
|
|
132
|
+
# phase, so accepting the same handoff twice is a no-op, and a restarted
|
|
133
|
+
# driver resumes where the last one stopped.
|
|
134
|
+
#
|
|
135
|
+
# Stall check: --stall-window (default 600 seconds, env PIPELINE_DRIVER_STALL_SECS).
|
|
136
|
+
# The stall starts at the later of the oldest in-flight phase's route time and
|
|
137
|
+
# the last lane status change, and is kept in driver/stall.json so a restart
|
|
138
|
+
# does not reset it. The lane state comes from `hq lanes list --json`
|
|
139
|
+
# (loop state and queue depth). The event fires once per stall;
|
|
140
|
+
# accepting any phase clears it. No notification hook exists for the driver,
|
|
141
|
+
# so firing writes the event and exits 28 for the parent's waiter.
|
|
142
|
+
#
|
|
143
|
+
# Start: loop lanes own their lifecycle; the driver reads their loop state.
|
|
144
|
+
#
|
|
145
|
+
# Env: PIPELINE_DRIVER_CONDUCTOR (default: pipeline-conductor.sh next to this
|
|
146
|
+
# script); PC_HQ (default: hq); every PC_* variable the conductor reads passes through.
|
|
147
|
+
#
|
|
148
|
+
# POSIX sh (dash-clean). Requires jq (JSON reads and small edits) and node
|
|
149
|
+
# (the state scan and stall check).
|
|
150
|
+
|
|
151
|
+
set -u
|
|
152
|
+
|
|
153
|
+
usage() { sed -n '3,/^# POSIX sh/p' "$0" >&2; exit 2; }
|
|
154
|
+
|
|
155
|
+
NL='
|
|
156
|
+
'
|
|
157
|
+
PRD=""; STATE=""; WT=""; SB=0; ALLOW_REPOS=0; TABLE=""; INTERVAL=15; MAX_FAILS=2; STALL="${PIPELINE_DRIVER_STALL_SECS:-600}"
|
|
158
|
+
while [ $# -gt 0 ]; do
|
|
159
|
+
case "$1" in
|
|
160
|
+
--prd|--state|--worktree|--interval|--max-phase-fails|--stall-window|--table)
|
|
161
|
+
[ $# -ge 2 ] || { echo "pipeline-driver: $1 needs a value" >&2; exit 2; } ;;
|
|
162
|
+
esac
|
|
163
|
+
case "$1" in
|
|
164
|
+
--prd) PRD="$2"; shift 2 ;;
|
|
165
|
+
--state) STATE="$2"; shift 2 ;;
|
|
166
|
+
--worktree) WT="${WT:+$WT$NL}$2"; shift 2 ;;
|
|
167
|
+
--story-branches) SB=1; shift ;;
|
|
168
|
+
--allow-repos-worktree) ALLOW_REPOS=1; shift ;;
|
|
169
|
+
--table) TABLE="$2"; shift 2 ;;
|
|
170
|
+
--interval) INTERVAL="$2"; shift 2 ;;
|
|
171
|
+
--max-phase-fails) MAX_FAILS="$2"; shift 2 ;;
|
|
172
|
+
--stall-window) STALL="$2"; shift 2 ;;
|
|
173
|
+
-h|--help) usage ;;
|
|
174
|
+
*) echo "pipeline-driver: unknown argument: $1" >&2; exit 2 ;;
|
|
175
|
+
esac
|
|
176
|
+
done
|
|
177
|
+
[ -n "$PRD" ] && [ -n "$STATE" ] || usage
|
|
178
|
+
[ -f "$PRD" ] || { echo "pipeline-driver: no such prd: $PRD" >&2; exit 2; }
|
|
179
|
+
case "$INTERVAL" in ''|*[!0-9.]*|.|*.*.*) echo "pipeline-driver: --interval must be a number of seconds" >&2; exit 2 ;; esac
|
|
180
|
+
case "$MAX_FAILS" in ''|*[!0-9]*|0) echo "pipeline-driver: --max-phase-fails must be a positive integer" >&2; exit 2 ;; esac
|
|
181
|
+
case "$STALL" in ''|*[!0-9]*|0) echo "pipeline-driver: --stall-window must be a positive number of seconds" >&2; exit 2 ;; esac
|
|
182
|
+
command -v jq >/dev/null 2>&1 || { echo "pipeline-driver: jq required" >&2; exit 2; }
|
|
183
|
+
command -v node >/dev/null 2>&1 || { echo "pipeline-driver: node required" >&2; exit 2; }
|
|
184
|
+
|
|
185
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
186
|
+
PC="${PIPELINE_DRIVER_CONDUCTOR:-$HERE/pipeline-conductor.sh}"
|
|
187
|
+
PC_HQ="${PC_HQ:-hq}"
|
|
188
|
+
export PC_HQ
|
|
189
|
+
mkdir -p "$STATE/stories" "$STATE/handoffs" "$STATE/envelopes" "$STATE/driver" || exit 2
|
|
190
|
+
STATE="$(cd "$STATE" && pwd)"
|
|
191
|
+
D="$STATE/driver"
|
|
192
|
+
LOG="$D/driver.log"
|
|
193
|
+
LOCK="$D/lock"
|
|
194
|
+
TAB="$(printf '\t')"
|
|
195
|
+
|
|
196
|
+
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >>"$LOG"; }
|
|
197
|
+
|
|
198
|
+
# ---- one copy per state dir ----
|
|
199
|
+
is_driver() { # pid -> 0 when that pid is alive and is a pipeline-driver
|
|
200
|
+
case "$1" in ''|*[!0-9]*) return 1 ;; esac
|
|
201
|
+
if kill -0 "$1" 2>/dev/null; then
|
|
202
|
+
ps -ww -o args= -p "$1" 2>/dev/null | grep -q 'pipeline-driver' && return 0
|
|
203
|
+
fi
|
|
204
|
+
# Native Windows Node writers store the Win32 PID; Git Bash ps/kill address
|
|
205
|
+
# MSYS PIDs. Translate an MSYS PID before querying the Windows process table,
|
|
206
|
+
# while retaining the direct lookup for native Node lock writers.
|
|
207
|
+
if command -v powershell.exe >/dev/null 2>&1; then
|
|
208
|
+
local winpid="$1"
|
|
209
|
+
if [ -r "/proc/$1/winpid" ]; then
|
|
210
|
+
winpid="$(cat "/proc/$1/winpid" 2>/dev/null || true)"
|
|
211
|
+
fi
|
|
212
|
+
case "$winpid" in ''|*[!0-9]*) return 1 ;; esac
|
|
213
|
+
powershell.exe -NoLogo -NoProfile -NonInteractive -Command \
|
|
214
|
+
"\$p = Get-CimInstance Win32_Process -Filter 'ProcessId = $winpid'; if (\$null -ne \$p) { [Console]::Out.Write(\$p.CommandLine) }" 2>/dev/null \
|
|
215
|
+
| grep -Eq 'pipeline-driver|core pipeline driver' && return 0
|
|
216
|
+
fi
|
|
217
|
+
return 1
|
|
218
|
+
}
|
|
219
|
+
OWN_LOCK=0
|
|
220
|
+
take_lock() {
|
|
221
|
+
if mkdir "$LOCK" 2>/dev/null; then
|
|
222
|
+
printf '%s\n' "$$" >"$LOCK/pid"; OWN_LOCK=1; return 0
|
|
223
|
+
fi
|
|
224
|
+
holder="$(cat "$LOCK/pid" 2>/dev/null || true)"
|
|
225
|
+
if is_driver "$holder"; then return 1; fi
|
|
226
|
+
log "STALE_LOCK pid=${holder:-none}; taking it over"
|
|
227
|
+
rm -rf "$LOCK"
|
|
228
|
+
mkdir "$LOCK" 2>/dev/null || return 1
|
|
229
|
+
printf '%s\n' "$$" >"$LOCK/pid"; OWN_LOCK=1
|
|
230
|
+
}
|
|
231
|
+
if ! take_lock; then
|
|
232
|
+
echo "pipeline-driver: another driver (pid $(cat "$LOCK/pid" 2>/dev/null)) runs against $STATE" >&2
|
|
233
|
+
log "REFUSED second copy pid=$$ holder=$(cat "$LOCK/pid" 2>/dev/null)"
|
|
234
|
+
exit 24
|
|
235
|
+
fi
|
|
236
|
+
printf '%s\n' "$$" >"$D/driver.pid"
|
|
237
|
+
rm -f "$D/exit"
|
|
238
|
+
|
|
239
|
+
cleanup() {
|
|
240
|
+
if [ "$OWN_LOCK" = 1 ]; then rm -rf "$LOCK"; rm -f "$D/driver.pid"; fi
|
|
241
|
+
}
|
|
242
|
+
finish() { # code reason
|
|
243
|
+
printf '%s %s\n' "$1" "$2" >"$D/exit"
|
|
244
|
+
log "EXIT $1 $2"
|
|
245
|
+
cleanup; OWN_LOCK=0
|
|
246
|
+
exit "$1"
|
|
247
|
+
}
|
|
248
|
+
# stop_run code reason: mark in-flight phases interrupted, withdraw unpicked envelopes, exit
|
|
249
|
+
stop_run() {
|
|
250
|
+
trap '' TERM INT
|
|
251
|
+
OUT="$("$PC" interrupt --state "$STATE" --note "$2" 2>&1)"
|
|
252
|
+
log "INTERRUPT rc=$?: $(printf '%s' "$OUT" | tr '\n' ' ')"
|
|
253
|
+
stop_lanes
|
|
254
|
+
rm -f "$D/stop.json"
|
|
255
|
+
finish "$1" "$2"
|
|
256
|
+
}
|
|
257
|
+
stop_lanes() {
|
|
258
|
+
[ -f "$STATE/lanes.json" ] || return 0
|
|
259
|
+
local deadline lane_file
|
|
260
|
+
lane_file="$STATE/lanes.json"
|
|
261
|
+
while IFS= read -r lane; do
|
|
262
|
+
[ -n "$lane" ] || continue
|
|
263
|
+
"$PC_HQ" lanes stop "$lane" >/dev/null 2>&1 || true
|
|
264
|
+
done <<EOF
|
|
265
|
+
$(jq -r 'to_entries[] | .value' "$STATE/lanes.json" 2>/dev/null)
|
|
266
|
+
EOF
|
|
267
|
+
deadline=$(( $(date +%s) + 60 ))
|
|
268
|
+
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
269
|
+
local rows
|
|
270
|
+
rows="$("$PC_HQ" lanes list --json 2>/dev/null)"
|
|
271
|
+
if printf '%s' "$rows" | jq -e --slurpfile owned "$lane_file" '
|
|
272
|
+
(if type == "array" then . else .lanes end) as $rows |
|
|
273
|
+
($rows | type == "array") and
|
|
274
|
+
all($owned[0][]; . as $id | ([ $rows[] | select(.lane_id == $id) ] | length == 0 or .[0].loop.state == "stopped"))' >/dev/null 2>&1; then
|
|
275
|
+
printf '{}\n' > "$lane_file"
|
|
276
|
+
return 0
|
|
277
|
+
fi
|
|
278
|
+
sleep 2
|
|
279
|
+
done
|
|
280
|
+
log "LANE_STOP_TIMEOUT unable to confirm all mapped lanes stopped or absent within 60s"
|
|
281
|
+
return 1
|
|
282
|
+
}
|
|
283
|
+
check_stop() {
|
|
284
|
+
[ -f "$D/stop.json" ] || return 0
|
|
285
|
+
stop_run 29 "stopped on request: $(tr '\n' ' ' <"$D/stop.json" 2>/dev/null)"
|
|
286
|
+
}
|
|
287
|
+
trap 'cleanup' EXIT
|
|
288
|
+
trap 'stop_run 143 "terminated"' TERM
|
|
289
|
+
trap 'stop_run 130 "interrupted"' INT
|
|
290
|
+
|
|
291
|
+
log "START pid=$$ prd=$PRD interval=${INTERVAL}s max_phase_fails=$MAX_FAILS stall_window=${STALL}s"
|
|
292
|
+
if [ -f "$D/stop.json" ]; then log "STALE_STOP removed driver/stop.json left from an earlier stop"; rm -f "$D/stop.json"; fi
|
|
293
|
+
# jq helpers: t = truthiness as JSON-writing tools treat it (null, false, 0,
|
|
294
|
+
# "", [], {} are false); ps = a value printed as text (null -> None).
|
|
295
|
+
JQ_DEFS='def t: . != null and . != false and . != 0 and . != "" and . != [] and . != {};
|
|
296
|
+
def ps: if . == null then "None" elif type == "string" then . elif . == true then "True" elif . == false then "False" else tojson end;'
|
|
297
|
+
# stories_jq <filter>: run <filter> over each readable, valid story file in
|
|
298
|
+
# byte-sorted name order; unreadable or invalid files are skipped
|
|
299
|
+
stories_jq() {
|
|
300
|
+
LC_ALL=C ls -A "$STATE/stories" 2>/dev/null | while IFS= read -r n; do
|
|
301
|
+
jq -rs "$JQ_DEFS if length == 1 then .[0] | ($1) else empty end" "$STATE/stories/$n" 2>/dev/null
|
|
302
|
+
done
|
|
303
|
+
}
|
|
304
|
+
REOPENED="$(stories_jq 'select(type == "object") | .reopen as $ro
|
|
305
|
+
| select(($ro | type) == "object" and ($ro.routed | t | not) and ($ro.done | t | not))
|
|
306
|
+
| "\(.id | ps)@\($ro.phase | ps)"' | tr '\n' ' ')"
|
|
307
|
+
[ -n "$REOPENED" ] && log "REOPENED_FIRST $REOPENED"
|
|
308
|
+
INTERRUPTED="$(stories_jq 'select(type == "object") | .interrupted as $it
|
|
309
|
+
| select(($it | type) == "object" and ($it.phase | t) and ($it.routed | t | not))
|
|
310
|
+
| "\(.id | ps)@\($it.phase | ps)"' | tr '\n' ' ')"
|
|
311
|
+
[ -n "$INTERRUPTED" ] && log "INTERRUPTED_FIRST $INTERRUPTED"
|
|
312
|
+
|
|
313
|
+
# ---- scan: read the state dir, print what to do, one tab-separated line each ----
|
|
314
|
+
# ARCHIVE <id> <phase> <handoff> a queued/held story still has a handoff for its current phase
|
|
315
|
+
# FAILCAP <id> <phase> <n> <why> that phase returned status failed in <n> >= max attempts
|
|
316
|
+
# (why: failed), or its engine exited early twice
|
|
317
|
+
# (why: engine_exited_early)
|
|
318
|
+
# EARLY <id> <phase> <secs> <reason> an in-flight phase's engine exited early
|
|
319
|
+
# ACCEPT <id> <phase> <handoff> an in-flight phase has its handoff
|
|
320
|
+
# RECHECK <id> a story is awaiting recheck
|
|
321
|
+
# DEADLINE <id> <phase> <deadline> an in-flight phase is past its deadline with no handoff
|
|
322
|
+
# INFLIGHT <id> <phase> <secs> an in-flight phase with no handoff, routed <secs> ago
|
|
323
|
+
# HELD <id> <decision-file> a story is held for an approval decision
|
|
324
|
+
# BLOCKED <id> <phase> <decision> a phase returned blocked (or cannot route); the owner decides
|
|
325
|
+
# AWAITING_GO <id> <decision> a story needs an explicit go
|
|
326
|
+
# INTERRUPTED <id> <phase> <at> a story was interrupted on a stop and is not routed again yet
|
|
327
|
+
# LANE_STOPPED <id> <phase> the in-flight phase's loop lane exited on a stop envelope
|
|
328
|
+
# ACTIVE <n> stories queued, in flight, held or awaiting recheck
|
|
329
|
+
scan() {
|
|
330
|
+
PD_STATE="$STATE" PD_MAX="$MAX_FAILS" PD_HQ="$PC_HQ" node - <<'JS'
|
|
331
|
+
'use strict';
|
|
332
|
+
const fs = require('fs'), path = require('path'), cp = require('child_process');
|
|
333
|
+
const S = process.env.PD_STATE, MAX = parseInt(process.env.PD_MAX, 10), HQ = process.env.PD_HQ;
|
|
334
|
+
const TERMINAL = ['passed', 'failed', 'blocked'];
|
|
335
|
+
const EARLY = 'engine_exited_early';
|
|
336
|
+
const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
337
|
+
const isNum = (v) => typeof v === 'number';
|
|
338
|
+
const truthy = (v) => !(v === null || v === undefined || v === false || v === 0 || v === '' ||
|
|
339
|
+
(Array.isArray(v) && !v.length) || (isObj(v) && !Object.keys(v).length));
|
|
340
|
+
const str = (v) => v === undefined || v === null ? 'None' : v === true ? 'True' : v === false ? 'False'
|
|
341
|
+
: typeof v === 'string' ? v : JSON.stringify(v);
|
|
342
|
+
const get = (o, k, d) => (isObj(o) && Object.prototype.hasOwnProperty.call(o, k) ? o[k] : d);
|
|
343
|
+
const byteSort = (a) => a.sort((x, y) => Buffer.compare(Buffer.from(x), Buffer.from(y)));
|
|
344
|
+
function load(p) {
|
|
345
|
+
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { return null; }
|
|
346
|
+
}
|
|
347
|
+
// whole seconds since the epoch of "YYYY-MM-DDTHH:MM:SS" (first 19 chars); null when unparseable
|
|
348
|
+
function epoch(ts) {
|
|
349
|
+
const m = /^(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{1,2}):(\d{1,2})$/.exec(String(ts).slice(0, 19));
|
|
350
|
+
if (!m) return null;
|
|
351
|
+
const [y, mo, d, h, mi, s] = m.slice(1).map(Number);
|
|
352
|
+
if (mo < 1 || mo > 12 || h > 23 || mi > 59 || s > 61) return null;
|
|
353
|
+
const ms = Date.UTC(y, mo - 1, d, h, mi, s);
|
|
354
|
+
if (d < 1 || new Date(Date.UTC(y, mo - 1, d)).getUTCDate() !== d) return null;
|
|
355
|
+
return Math.floor(ms / 1000);
|
|
356
|
+
}
|
|
357
|
+
const now = () => Date.now() / 1000;
|
|
358
|
+
const isoNow = () => new Date().toISOString().slice(0, 19) + 'Z';
|
|
359
|
+
const collapse = (s) => Array.from(s.split(/\s+/).filter(Boolean).join(' ')).slice(0, 300).join('');
|
|
360
|
+
let lanes = null;
|
|
361
|
+
function laneRows() {
|
|
362
|
+
if (lanes === null) {
|
|
363
|
+
lanes = new Map();
|
|
364
|
+
try {
|
|
365
|
+
const owned = load(path.join(S, 'lanes.json')) || {};
|
|
366
|
+
const r = cp.spawnSync(HQ, ['lanes', 'list', '--json'], { encoding: 'utf8', timeout: 30000 });
|
|
367
|
+
if (!r.error) {
|
|
368
|
+
const text = String(r.stdout || ''), starts = [text.indexOf('['), text.indexOf('{')].filter((n) => n >= 0);
|
|
369
|
+
const parsed = JSON.parse(starts.length ? text.slice(Math.min(...starts)) : text);
|
|
370
|
+
const rows = Array.isArray(parsed) ? parsed : (isObj(parsed) && Array.isArray(parsed.lanes) ? parsed.lanes : []);
|
|
371
|
+
for (const x of rows) if (isObj(x) && Object.values(owned).includes(get(x, 'lane_id', null))) lanes.set(get(x, 'lane_id', null), x);
|
|
372
|
+
}
|
|
373
|
+
} catch (e) { /* no lanes */ }
|
|
374
|
+
}
|
|
375
|
+
return lanes;
|
|
376
|
+
}
|
|
377
|
+
function laneRow(worker) {
|
|
378
|
+
const owned = load(path.join(S, 'lanes.json')) || {};
|
|
379
|
+
return laneRows().get(owned[worker]) || null;
|
|
380
|
+
}
|
|
381
|
+
// journal events of a lane containing <needle>, filtered to those written after <since>
|
|
382
|
+
function journal(worker, needle, since, pick) {
|
|
383
|
+
const rd = laneDir(worker);
|
|
384
|
+
if (!truthy(rd)) return undefined;
|
|
385
|
+
let lines;
|
|
386
|
+
try { lines = fs.readFileSync(path.join(String(rd), 'journal.jsonl'), 'utf8').split('\n'); } catch (e) { return undefined; }
|
|
387
|
+
let found = null;
|
|
388
|
+
for (const line of lines) {
|
|
389
|
+
if (!line.includes(needle)) continue;
|
|
390
|
+
let ev;
|
|
391
|
+
try { ev = JSON.parse(line); } catch (e) { continue; }
|
|
392
|
+
if (!pick(ev)) continue;
|
|
393
|
+
const ts = get(ev, 'ts', null);
|
|
394
|
+
if (since !== null && typeof ts === 'string') {
|
|
395
|
+
const t = epoch(ts);
|
|
396
|
+
if (t !== null && t < since - 1) continue;
|
|
397
|
+
}
|
|
398
|
+
found = ev;
|
|
399
|
+
}
|
|
400
|
+
return found;
|
|
401
|
+
}
|
|
402
|
+
function laneStopped(worker, since) {
|
|
403
|
+
const row = laneRow(worker);
|
|
404
|
+
return !!row && get(get(row, 'loop', {}), 'state') === 'stopped';
|
|
405
|
+
}
|
|
406
|
+
function phaseExit(worker, sid, ph, since) {
|
|
407
|
+
// the lane's last phase-exit event for this phase written after it was routed
|
|
408
|
+
const row = laneRow(worker);
|
|
409
|
+
if (!row) return null;
|
|
410
|
+
const state = get(get(row, 'loop', {}), 'state');
|
|
411
|
+
if (state === 'parked') return { reason: 'loop_stalled', phase: ph, story_id: sid };
|
|
412
|
+
return state === 'failed' ? { reason: state, phase: ph, story_id: sid } : null;
|
|
413
|
+
}
|
|
414
|
+
function mtime(p) {
|
|
415
|
+
try { return fs.statSync(p).mtimeMs / 1000; } catch (e) { return null; }
|
|
416
|
+
}
|
|
417
|
+
const out = []; let active = 0;
|
|
418
|
+
const sd = path.join(S, 'stories');
|
|
419
|
+
for (const n of byteSort(fs.readdirSync(sd))) {
|
|
420
|
+
if (!n.endsWith('.json')) continue;
|
|
421
|
+
const st = load(path.join(sd, n));
|
|
422
|
+
if (!isObj(st) || !('id' in st)) continue;
|
|
423
|
+
const sid = st.id, state = get(st, 'state', null);
|
|
424
|
+
if (['queued', 'in_flight', 'held', 'awaiting_recheck', 'interrupted'].includes(state)) active++;
|
|
425
|
+
const it = get(st, 'interrupted', null);
|
|
426
|
+
if (isObj(it) && truthy(it.phase) && !truthy(it.routed)) out.push(['INTERRUPTED', sid, str(it.phase), str(get(it, 'at', ''))]);
|
|
427
|
+
const phases = truthy(st.phases) ? st.phases : [];
|
|
428
|
+
const cur = get(st, 'current', 0);
|
|
429
|
+
if (!(Number.isInteger(cur) && cur >= 0 && cur < phases.length)) continue;
|
|
430
|
+
const ph = phases[cur].phase;
|
|
431
|
+
const worker = phases[cur].worker;
|
|
432
|
+
const h = path.join(S, 'handoffs', `${sid}-${ph}.json`);
|
|
433
|
+
if (state === 'queued' || state === 'held') {
|
|
434
|
+
if (fs.existsSync(h)) out.push(['ARCHIVE', sid, ph, h]);
|
|
435
|
+
// count attempts whose handoff said failed; blocked ones are the owner's, not a retry
|
|
436
|
+
const hd = path.join(S, 'handoffs');
|
|
437
|
+
// a retry by the owner starts the count over: archives up to fail_ack are acknowledged
|
|
438
|
+
const fa = get(st, 'fail_ack', null);
|
|
439
|
+
const ack = isObj(fa) ? Math.trunc(Number(get(fa, ph, 0) || 0)) : 0;
|
|
440
|
+
const pre = `${sid}-${ph}.failed.`;
|
|
441
|
+
const tried = fs.readdirSync(hd).filter((f) => {
|
|
442
|
+
if (!f.startsWith(pre) || !f.endsWith('.json')) return false;
|
|
443
|
+
const mid = f.slice(pre.length, -5);
|
|
444
|
+
return /^\d+$/.test(mid) && parseInt(mid, 10) > ack;
|
|
445
|
+
}).map((f) => path.join(hd, f));
|
|
446
|
+
if (fs.existsSync(h)) tried.push(h);
|
|
447
|
+
const hs = tried.map((f) => { const x = load(f); return isObj(x) ? x : {}; });
|
|
448
|
+
const fails = hs.filter((x) => x.status === 'failed').length;
|
|
449
|
+
const early = hs.filter((x) => x.status === 'failed' && x.exit_reason === EARLY).length;
|
|
450
|
+
if (early >= 2) out.push(['FAILCAP', sid, ph, String(early), EARLY]);
|
|
451
|
+
else if (fails >= MAX) out.push(['FAILCAP', sid, ph, String(fails), 'failed']);
|
|
452
|
+
if (state === 'held') {
|
|
453
|
+
const dp = path.join(S, 'decisions', `${sid}-${get(st, 'held_for', worker)}.md`);
|
|
454
|
+
out.push(['HELD', sid, dp]);
|
|
455
|
+
}
|
|
456
|
+
} else if (state === 'in_flight') {
|
|
457
|
+
const hd = load(h);
|
|
458
|
+
let since = get(st, 'routed_at', null);
|
|
459
|
+
if (!isNum(since)) since = mtime(path.join(S, 'envelopes', `${sid}-${ph}.json`));
|
|
460
|
+
const mine = isObj(hd) && hd.phase === ph;
|
|
461
|
+
const hstat = mine ? get(hd, 'status', null) : null;
|
|
462
|
+
let ev = null;
|
|
463
|
+
if (mine && TERMINAL.includes(hstat) && get(hd, 'exit_reason', null) !== EARLY) {
|
|
464
|
+
out.push(['ACCEPT', sid, ph, h]);
|
|
465
|
+
} else if (mine && get(hd, 'exit_reason', null) === EARLY) {
|
|
466
|
+
let secs = get(hd, 'elapsed_s', null);
|
|
467
|
+
if (!isNum(secs)) secs = since !== null ? Math.trunc(now() - since) : 0;
|
|
468
|
+
const why = collapse(str(truthy(hd.notes) ? hd.notes : truthy(hd.summary) ? hd.summary : 'no reason given'));
|
|
469
|
+
out.push(['EARLY', sid, ph, String(Math.trunc(secs)), why]);
|
|
470
|
+
} else if ((!mine || !TERMINAL.includes(hstat)) && (ev = phaseExit(worker, sid, ph, since)) !== null) {
|
|
471
|
+
let secs = get(ev, 'elapsed_s', null);
|
|
472
|
+
if (!isNum(secs)) secs = since !== null ? Math.trunc(now() - since) : 0;
|
|
473
|
+
const why = collapse(str(truthy(ev.reason) ? ev.reason : 'no reason given'));
|
|
474
|
+
if (get(ev, 'reason', null) === 'loop_stalled') out.push(['LANE_STALLED', sid, ph, worker]);
|
|
475
|
+
else out.push(['EARLY', sid, ph, String(Math.trunc(secs)), why]);
|
|
476
|
+
} else if (laneStopped(worker, since)) {
|
|
477
|
+
out.push(['LANE_STOPPED', sid, ph]);
|
|
478
|
+
} else {
|
|
479
|
+
const env = load(path.join(S, 'envelopes', `${sid}-${ph}.json`));
|
|
480
|
+
const dl = get(env, 'deadline', null);
|
|
481
|
+
if (typeof dl === 'string' && isoNow() > dl) out.push(['DEADLINE', sid, ph, dl]);
|
|
482
|
+
if (since !== null) out.push(['INFLIGHT', sid, ph, String(Math.max(0, Math.trunc(now() - since)))]);
|
|
483
|
+
}
|
|
484
|
+
} else if (state === 'awaiting_go') {
|
|
485
|
+
out.push(['AWAITING_GO', sid, path.join(S, 'decisions', `${sid}-go.md`)]);
|
|
486
|
+
} else if (state === 'blocked_needs_owner') {
|
|
487
|
+
out.push(['BLOCKED', sid, ph, path.join(S, 'decisions', `${sid}-blocked-${ph}.md`)]);
|
|
488
|
+
} else if (state === 'awaiting_recheck') {
|
|
489
|
+
out.push(['RECHECK', sid]);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
out.push(['ACTIVE', String(active)]);
|
|
493
|
+
process.stdout.write(out.map((r) => r.join('\t') + '\n').join(''));
|
|
494
|
+
JS
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
# archive <id> <phase> <handoff>: move a stale handoff aside under the next free number
|
|
498
|
+
archive() {
|
|
499
|
+
n=1
|
|
500
|
+
while [ -e "$STATE/handoffs/$1-$2.failed.$n.json" ]; do n=$((n + 1)); done
|
|
501
|
+
mv -f "$3" "$STATE/handoffs/$1-$2.failed.$n.json" && log "ARCHIVE $1 $2 -> $1-$2.failed.$n.json"
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
# early_handoff <id> <phase> <secs> <reason>: make <id>-<phase>.json the failed
|
|
505
|
+
# handoff of an early engine exit (exit_reason engine_exited_early), keeping a
|
|
506
|
+
# lane-written one's fields; print its path
|
|
507
|
+
early_handoff() {
|
|
508
|
+
PD_H="$STATE/handoffs/$1-$2.json" PD_SID="$1" PD_PH="$2" PD_SECS="$3" PD_WHY="$4" \
|
|
509
|
+
ehp="$STATE/handoffs/$1-$2.json"
|
|
510
|
+
ehw="$(jq -rs "$JQ_DEFS"' if length == 1 then .[0].phases[.[0].current].worker | ps else empty end' \
|
|
511
|
+
"$STATE/stories/$1.json" 2>/dev/null)"
|
|
512
|
+
ehh="$(jq -cs 'if length == 1 then .[0] else null end' "$ehp" 2>/dev/null)" || ehh=null
|
|
513
|
+
[ -n "$ehh" ] || ehh=null
|
|
514
|
+
printf '%s\n' "$ehh" | jq -c --arg sid "$1" --arg ph "$2" --arg secs "$3" --arg why "$4" --arg worker "$ehw" \
|
|
515
|
+
"$JQ_DEFS"' (if type == "object" and .phase == $ph then . else {} end)
|
|
516
|
+
| ($secs | tonumber | if . < 0 then -(-. | floor) else floor end) as $el
|
|
517
|
+
| . + {schema: "hq-phase-handoff/v1", story_id: $sid, phase: $ph,
|
|
518
|
+
worker_id: (if (.worker_id | t) then .worker_id else $worker end), status: "failed",
|
|
519
|
+
exit_reason: "engine_exited_early", elapsed_s: $el}
|
|
520
|
+
| if ((if has("summary") then .summary else "" end) | ps | startswith("engine_exited_early")) then .
|
|
521
|
+
else .summary = "engine_exited_early after \($secs)s: \($why)" end
|
|
522
|
+
| if has("files_changed") then . else .files_changed = [] end
|
|
523
|
+
| if has("commits") then . else .commits = [] end
|
|
524
|
+
| if has("back_pressure") then . else .back_pressure = {tests: "skip", lint: "skip", typecheck: "skip", build: "skip"} end
|
|
525
|
+
| if has("context_for_next") then . else .context_for_next = "" end
|
|
526
|
+
| if has("notes") then . else .notes = $why end' >"$ehp.tmp" && mv -f "$ehp.tmp" "$ehp" && printf '%s\n' "$ehp"
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
# stall_check: print "FIRE <since>" the first time the stall condition has held
|
|
530
|
+
# for --stall-window; keeps driver/stall.json and appends the event to
|
|
531
|
+
# <state>/events.jsonl when it fires
|
|
532
|
+
stall_check() {
|
|
533
|
+
PD_STATE="$STATE" PD_WINDOW="$STALL" PD_HQ="$PC_HQ" node - <<'JS'
|
|
534
|
+
'use strict';
|
|
535
|
+
const fs = require('fs'), path = require('path');
|
|
536
|
+
const S = process.env.PD_STATE, W = parseInt(process.env.PD_WINDOW, 10), HQ = process.env.PD_HQ;
|
|
537
|
+
const SF = path.join(S, 'driver', 'stall.json');
|
|
538
|
+
const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
539
|
+
const truthy = (v) => !(v === null || v === undefined || v === false || v === 0 || v === '' ||
|
|
540
|
+
(Array.isArray(v) && !v.length) || (isObj(v) && !Object.keys(v).length));
|
|
541
|
+
const get = (o, k, d) => (isObj(o) && Object.prototype.hasOwnProperty.call(o, k) ? o[k] : d);
|
|
542
|
+
const byteSort = (a) => a.sort((x, y) => Buffer.compare(Buffer.from(x), Buffer.from(y)));
|
|
543
|
+
const cmp = (x, y) => (x < y ? -1 : x > y ? 1 : 0);
|
|
544
|
+
function load(p) {
|
|
545
|
+
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { return null; }
|
|
546
|
+
}
|
|
547
|
+
function epoch(iso) {
|
|
548
|
+
const m = /^(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{1,2}):(\d{1,2})$/.exec(String(iso).slice(0, 19));
|
|
549
|
+
if (!m) return null;
|
|
550
|
+
const [y, mo, d, h, mi, s] = m.slice(1).map(Number);
|
|
551
|
+
if (mo < 1 || mo > 12 || h > 23 || mi > 59 || s > 61) return null;
|
|
552
|
+
if (d < 1 || new Date(Date.UTC(y, mo - 1, d)).getUTCDate() !== d) return null;
|
|
553
|
+
return Math.floor(Date.UTC(y, mo - 1, d, h, mi, s) / 1000);
|
|
554
|
+
}
|
|
555
|
+
function files(d) {
|
|
556
|
+
try { return fs.readdirSync(d).filter((n) => !n.startsWith('.')); } catch (e) { return []; }
|
|
557
|
+
}
|
|
558
|
+
const now = () => Date.now() / 1000;
|
|
559
|
+
const iso = (secs) => new Date(Math.floor(secs) * 1000).toISOString().slice(0, 19) + 'Z';
|
|
560
|
+
// Python json.dumps default output: ", " / ": " separators, non-ASCII escaped
|
|
561
|
+
function dumps(v) {
|
|
562
|
+
if (v === null || v === undefined) return 'null';
|
|
563
|
+
if (v === true) return 'true';
|
|
564
|
+
if (v === false) return 'false';
|
|
565
|
+
if (typeof v === 'number') {
|
|
566
|
+
let s = String(v);
|
|
567
|
+
const m = /^(-?)(\d)(?:\.(\d+))?e([+-])(\d+)$/.exec(s);
|
|
568
|
+
if (m) s = m[1] + m[2] + (m[3] ? '.' + m[3] : '') + 'e' + m[4] + m[5].padStart(2, '0');
|
|
569
|
+
return s;
|
|
570
|
+
}
|
|
571
|
+
if (typeof v === 'string') return JSON.stringify(v).replace(/[\u0080-]/g, (c) => '\\u' + c.charCodeAt(0).toString(16).padStart(4, '0'));
|
|
572
|
+
if (Array.isArray(v)) return '[' + v.map(dumps).join(', ') + ']';
|
|
573
|
+
return '{' + Object.keys(v).map((k) => dumps(k) + ': ' + dumps(v[k])).join(', ') + '}';
|
|
574
|
+
}
|
|
575
|
+
let rows = [];
|
|
576
|
+
try {
|
|
577
|
+
const owned = load(path.join(S, 'lanes.json')) || {};
|
|
578
|
+
const r = require('child_process').spawnSync(HQ, ['lanes', 'list', '--json'], { encoding: 'utf8', timeout: 30000 });
|
|
579
|
+
const text = String(r.stdout || ''), starts = [text.indexOf('['), text.indexOf('{')].filter((n) => n >= 0);
|
|
580
|
+
const parsed = JSON.parse(starts.length ? text.slice(Math.min(...starts)) : text);
|
|
581
|
+
const listed = Array.isArray(parsed) ? parsed : (isObj(parsed) && Array.isArray(parsed.lanes) ? parsed.lanes : []);
|
|
582
|
+
rows = listed.filter((row) => isObj(row) && Object.values(owned).includes(get(row, 'lane_id', null)));
|
|
583
|
+
} catch (e) { process.exit(0); }
|
|
584
|
+
const inflight = [];
|
|
585
|
+
const sd = path.join(S, 'stories');
|
|
586
|
+
for (const n of byteSort(fs.readdirSync(sd))) {
|
|
587
|
+
const st = n.endsWith('.json') ? load(path.join(sd, n)) : null;
|
|
588
|
+
if (!isObj(st) || st.state !== 'in_flight') continue;
|
|
589
|
+
const phases = truthy(st.phases) ? st.phases : [];
|
|
590
|
+
const cur = get(st, 'current', 0);
|
|
591
|
+
if (!(Number.isInteger(cur) && cur >= 0 && cur < phases.length)) continue;
|
|
592
|
+
let since = get(st, 'routed_at', null);
|
|
593
|
+
if (typeof since !== 'number') {
|
|
594
|
+
try { since = fs.statSync(path.join(S, 'envelopes', `${st.id}-${phases[cur].phase}.json`)).mtimeMs / 1000; } catch (e) { since = now(); }
|
|
595
|
+
}
|
|
596
|
+
inflight.push([st.id, phases[cur].worker, since]);
|
|
597
|
+
}
|
|
598
|
+
const live = rows.filter((r) => isObj(r) && isObj(r.loop));
|
|
599
|
+
let cond = false, stories = [], lanes = [], since = null;
|
|
600
|
+
if (inflight.length && live.length && live.every((r) => r.loop.state === 'waiting' && r.loop.queue_depth === 0)) {
|
|
601
|
+
cond = true;
|
|
602
|
+
stories = inflight.map((x) => x[0]); lanes = live.map((r) => get(r, 'lane_id', null)).sort(cmp);
|
|
603
|
+
const changed = live.filter((r) => truthy(r.loop.updated_at)).map((r) => epoch(r.loop.updated_at));
|
|
604
|
+
since = Math.max(Math.min(...inflight.map((x) => x[2])), ...changed.filter((c) => c !== null));
|
|
605
|
+
} else {
|
|
606
|
+
// a routed phase sits in its lane queue and the lane has nothing active
|
|
607
|
+
const owned = load(path.join(S, 'lanes.json')) || {};
|
|
608
|
+
const byWorker = new Map();
|
|
609
|
+
for (const r of live) for (const [worker, lane] of Object.entries(owned)) if (lane === get(r, 'lane_id', null)) byWorker.set(worker, r);
|
|
610
|
+
for (const [sid, worker, s] of inflight) {
|
|
611
|
+
const r = byWorker.get(worker);
|
|
612
|
+
if (r && get(r.loop, 'queue_depth', 0) > 0 && !truthy(get(r.loop, 'active_envelope_id', null))) {
|
|
613
|
+
cond = true; stories.push(sid); lanes.push(get(r, 'lane_id', worker));
|
|
614
|
+
since = since === null ? s : Math.min(since, s);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
const prev = load(SF);
|
|
619
|
+
if (!cond) {
|
|
620
|
+
if (isObj(prev) && !truthy(prev.fired)) fs.unlinkSync(SF);
|
|
621
|
+
process.exit(0);
|
|
622
|
+
}
|
|
623
|
+
if (isObj(prev)) {
|
|
624
|
+
if (truthy(prev.fired)) process.exit(0);
|
|
625
|
+
since = get(prev, 'since_epoch', since);
|
|
626
|
+
}
|
|
627
|
+
const sinceIso = iso(since);
|
|
628
|
+
const rec = { since_epoch: since, since: sinceIso, stories, lanes: [...new Set(lanes)].sort(cmp), fired: false };
|
|
629
|
+
if (now() - since >= W) {
|
|
630
|
+
rec.fired = true;
|
|
631
|
+
fs.appendFileSync(path.join(S, 'events.jsonl'), dumps({ event: 'stall', stories, lanes: rec.lanes, since: sinceIso,
|
|
632
|
+
at: iso(now()) }) + '\n');
|
|
633
|
+
process.stdout.write('FIRE ' + sinceIso + ' stories=' + stories.join(',') + ' lanes=' + rec.lanes.join(',') + '\n');
|
|
634
|
+
}
|
|
635
|
+
fs.writeFileSync(SF + '.tmp', dumps(rec));
|
|
636
|
+
fs.renameSync(SF + '.tmp', SF);
|
|
637
|
+
JS
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
# run_pc <args...>: run the conductor, leaving stdout in $OUT and rc in $RC
|
|
641
|
+
run_pc() {
|
|
642
|
+
OUT="$("$PC" "$@" 2>"$D/.pc.err")"; RC=$?
|
|
643
|
+
ERR="$(cat "$D/.pc.err" 2>/dev/null)"; rm -f "$D/.pc.err"
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
# go_list: the stories awaiting an explicit go, from a read-only scan
|
|
647
|
+
go_list() {
|
|
648
|
+
scan | while IFS="$TAB" read -r a b c d; do
|
|
649
|
+
[ "$a" = AWAITING_GO ] && printf '%s ' "$b"
|
|
650
|
+
done | sed 's/ $//'
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
# run_tick: one conductor tick with every --worktree and --story-branches passed through
|
|
654
|
+
run_tick() {
|
|
655
|
+
set -- tick --prd "$PRD" --state "$STATE"
|
|
656
|
+
[ "$SB" = 1 ] && set -- "$@" --story-branches
|
|
657
|
+
[ "$ALLOW_REPOS" = 1 ] && set -- "$@" --allow-repos-worktree
|
|
658
|
+
[ -n "$TABLE" ] && set -- "$@" --table "$TABLE"
|
|
659
|
+
if [ -n "$WT" ]; then
|
|
660
|
+
oldifs="$IFS"; IFS="$NL"; set -f
|
|
661
|
+
for w in $WT; do set -- "$@" --worktree "$w"; done
|
|
662
|
+
set +f; IFS="$oldifs"
|
|
663
|
+
fi
|
|
664
|
+
run_pc "$@"
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
# ---- act on one scan; sets ACTED=1 when the state dir changed, PENDING_EXIT on a stop ----
|
|
668
|
+
act() {
|
|
669
|
+
ACTED=0; ACTIVE=""; HELD_ITEM=""; GATE_DUE=0; BLOCKED_ITEMS=""; GO_ITEMS=""; INT_ITEMS=""
|
|
670
|
+
while IFS="$TAB" read -r a b c d e; do
|
|
671
|
+
case "$a" in
|
|
672
|
+
ARCHIVE) archive "$b" "$c" "$d"; ACTED=1 ;;
|
|
673
|
+
FAILCAP)
|
|
674
|
+
log "FAILCAP $b $c ${e:-failed} in $d attempt(s)"
|
|
675
|
+
run_pc failcap --state "$STATE" --story "$b"
|
|
676
|
+
log "FAILCAP_HOLD $b $c rc=$RC: $OUT${ERR:+ | $ERR}"
|
|
677
|
+
case "$RC:$OUT" in
|
|
678
|
+
10:PHASE_FAILCAP*|10:ALREADY*) ;;
|
|
679
|
+
*) finish 23 "failcap $b $c: unexpected (rc=$RC): $OUT $ERR" ;;
|
|
680
|
+
esac
|
|
681
|
+
[ "$e" = engine_exited_early ] && finish 21 "phase $b/$c: engine_exited_early in $d of $d attempt(s); the lane was restarted once in place and the engine exited early again; held for the owner (decision: ${OUT##* }); the parent decides whether to retry, change the story, or stop"
|
|
682
|
+
fail_reason="$(jq -r '.blocked.text // "" | gsub("[\\r\\n\\t]+"; " ")' "$STATE/stories/$b.json" 2>/dev/null)"
|
|
683
|
+
finish 21 "phase $b/$c returned handoff status failed in $d of $d attempt(s) (limit $MAX_FAILS); reason: $fail_reason; held for the owner (decision: ${OUT##* }); the parent decides whether to retry, change the story, or stop" ;;
|
|
684
|
+
EARLY)
|
|
685
|
+
log "TICK: EARLY_EXIT $b/$c after ${d}s: engine_exited_early ($e)"
|
|
686
|
+
h="$(early_handoff "$b" "$c" "$d" "$e")" || finish 23 "could not write the early-exit handoff for $b/$c"
|
|
687
|
+
rm -f "$D/stall.json"
|
|
688
|
+
run_pc accept --state "$STATE" --story "$b" --handoff "$h"
|
|
689
|
+
log "ACCEPT $b $c rc=$RC: $OUT${ERR:+ | $ERR}"
|
|
690
|
+
case "$RC:$OUT" in
|
|
691
|
+
1:PHASE_FAILED*) ACTED=1 ;;
|
|
692
|
+
*) finish 23 "accept early exit $b $c: unexpected (rc=$RC): $OUT $ERR" ;;
|
|
693
|
+
esac ;;
|
|
694
|
+
ACCEPT)
|
|
695
|
+
rm -f "$D/stall.json"
|
|
696
|
+
log "ACCEPTING $b $c $d"
|
|
697
|
+
run_pc accept --state "$STATE" --story "$b" --handoff "$d"
|
|
698
|
+
log "ACCEPT $b $c rc=$RC: $OUT${ERR:+ | $ERR}"
|
|
699
|
+
case "$RC:$OUT" in
|
|
700
|
+
0:NEXT*|0:RECHECK*|1:PHASE_*|10:PHASE_BLOCKED*) ACTED=1 ;;
|
|
701
|
+
*) finish 23 "accept $b $c: unexpected (rc=$RC): $OUT $ERR" ;;
|
|
702
|
+
esac ;;
|
|
703
|
+
RECHECK)
|
|
704
|
+
log "RECHECKING $b"
|
|
705
|
+
run_pc recheck --prd "$PRD" --state "$STATE" --story "$b"
|
|
706
|
+
log "RECHECK $b rc=$RC: $(printf '%s' "$OUT" | tr '\n' ' ')${ERR:+ | $ERR}"
|
|
707
|
+
case "$RC:$OUT" in
|
|
708
|
+
0:VERIFIED*) ACTED=1; case "$OUT" in *RUN_GATE*) GATE_DUE=1 ;; esac ;;
|
|
709
|
+
1:ROUTED_BACK*|1:FAILED*) ACTED=1 ;;
|
|
710
|
+
*) finish 23 "recheck $b: unexpected (rc=$RC): $OUT $ERR" ;;
|
|
711
|
+
esac ;;
|
|
712
|
+
DEADLINE) finish 22 "phase $b/$c passed its deadline $d with no handoff" ;;
|
|
713
|
+
HELD) HELD_ITEM="$b $c" ;;
|
|
714
|
+
BLOCKED)
|
|
715
|
+
blocked_reason="$(jq -r '.blocked.text // "" | gsub("[\\r\\n\\t]+"; " ")' "$STATE/stories/$b.json" 2>/dev/null)"
|
|
716
|
+
BLOCKED_ITEMS="${BLOCKED_ITEMS:+$BLOCKED_ITEMS; }$b/$c${blocked_reason:+: $blocked_reason} (decision: $d)" ;;
|
|
717
|
+
AWAITING_GO) GO_ITEMS="${GO_ITEMS:+$GO_ITEMS, }$b (decision: $c)" ;;
|
|
718
|
+
INTERRUPTED) INT_ITEMS="${INT_ITEMS:+$INT_ITEMS, }$b at $c ($d)" ;;
|
|
719
|
+
LANE_STOPPED)
|
|
720
|
+
run_pc interrupt --state "$STATE" --story "$b" --note "the $c lane exited on a stop envelope"
|
|
721
|
+
log "LANE_STOPPED $b $c rc=$RC: $OUT${ERR:+ | $ERR}"
|
|
722
|
+
ACTED=1 ;;
|
|
723
|
+
LANE_STALLED)
|
|
724
|
+
company="$(jq -r '.company_slug // empty' "$STATE/run.json" 2>/dev/null)"
|
|
725
|
+
lane="$(jq -r --arg w "$d" '.[$w] // empty' "$STATE/lanes.json" 2>/dev/null)"
|
|
726
|
+
questions="$("$PC_HQ" lanes questions list --company "$company" --json 2>/dev/null | jq -c --arg lane "$lane" '.questions[]? | select(.lane_id == $lane and .status == "pending")' 2>/dev/null | head -1)"
|
|
727
|
+
log "LANE_QUESTION story=$b phase=$c lane=$lane question=${questions:-missing}"
|
|
728
|
+
finish 28 "loop lane $lane parked after repeated stalls for $b/$c; answer its pending question with hq lanes questions answer, then restart the driver: ${questions:-no pending question returned}" ;;
|
|
729
|
+
INFLIGHT) ;;
|
|
730
|
+
ACTIVE) ACTIVE="$b" ;;
|
|
731
|
+
'') ;;
|
|
732
|
+
*) finish 23 "scan printed an unknown line: $a" ;;
|
|
733
|
+
esac
|
|
734
|
+
done <<EOF
|
|
735
|
+
$(scan)
|
|
736
|
+
EOF
|
|
737
|
+
[ -n "$ACTIVE" ] || finish 23 "could not read the state dir $STATE"
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
while :; do
|
|
741
|
+
check_stop
|
|
742
|
+
# Settle the state dir: archive, accept and recheck until a scan finds nothing to do.
|
|
743
|
+
i=0
|
|
744
|
+
while :; do
|
|
745
|
+
act
|
|
746
|
+
[ "$GATE_DUE" = 1 ] && finish 20 "regression gate due"
|
|
747
|
+
[ "$ACTED" = 1 ] || break
|
|
748
|
+
i=$((i + 1))
|
|
749
|
+
[ "$i" -lt 100 ] || finish 23 "state did not settle after 100 scans"
|
|
750
|
+
done
|
|
751
|
+
[ -n "$HELD_ITEM" ] && finish 21 "story held for approval: $HELD_ITEM"
|
|
752
|
+
FIRE="$(stall_check)"
|
|
753
|
+
if [ -n "$FIRE" ]; then
|
|
754
|
+
# FIRE <since> stories=<ids> lanes=<ids>
|
|
755
|
+
set -- $FIRE
|
|
756
|
+
log "STALL since $2 $3 $4: no phase accepted for the whole ${STALL}s stall window; event in $STATE/events.jsonl"
|
|
757
|
+
finish 28 "stall: since $2, $3 $4, no phase accepted for ${STALL}s and no lane is working on them (event: $STATE/events.jsonl); check the lanes, then restart the driver"
|
|
758
|
+
fi
|
|
759
|
+
|
|
760
|
+
check_stop
|
|
761
|
+
INT_NOW="$INT_ITEMS"
|
|
762
|
+
run_tick
|
|
763
|
+
GO_NOW="$(go_list)"
|
|
764
|
+
if [ -n "$OUT" ] || [ "$GO_NOW" != "${GO_LAST:-}" ] || [ -n "$INT_NOW" ]; then
|
|
765
|
+
log "TICK: $(printf '%s' "$OUT" | tr '\n' ' ')${GO_NOW:+| awaiting go: $GO_NOW}${INT_NOW:+| interrupted, routing first: $INT_NOW}"
|
|
766
|
+
fi
|
|
767
|
+
GO_LAST="$GO_NOW"
|
|
768
|
+
case "$RC" in 0|3|11|12) ;; *) finish 23 "tick failed (rc=$RC): $ERR" ;; esac
|
|
769
|
+
HELD_ITEM=""; GATE=""; LANE_DOWN=""; NO_LANE=""
|
|
770
|
+
while IFS= read -r line; do
|
|
771
|
+
case "$line" in
|
|
772
|
+
''|ROUTED\ *|RETRY\ *|PARKED_DEPENDENCY\ *|RELEASED_DEPENDENCY\ *) ;;
|
|
773
|
+
CUT\ *|REUSED\ *|EXISTS\ *|MAX_STORIES\ *|BLOCKED_OWNER\ *|AWAITING_GO\ *) ;;
|
|
774
|
+
HELD\ *) HELD_ITEM="${line#HELD }" ;;
|
|
775
|
+
LANE_DOWN\ *) [ -n "$LANE_DOWN" ] || LANE_DOWN="${line#LANE_DOWN }" ;;
|
|
776
|
+
NO_LANE\ *) [ -n "$NO_LANE" ] || NO_LANE="${line#NO_LANE }" ;;
|
|
777
|
+
RUN_GATE) GATE=due ;;
|
|
778
|
+
GATE_FAILED) GATE=failed ;;
|
|
779
|
+
*) finish 23 "tick printed an unexpected line: $line" ;;
|
|
780
|
+
esac
|
|
781
|
+
done <<EOF
|
|
782
|
+
$OUT
|
|
783
|
+
EOF
|
|
784
|
+
if [ -n "$LANE_DOWN" ]; then
|
|
785
|
+
# LANE_DOWN <id> <phase> <worker> lanes-<code>
|
|
786
|
+
set -- $LANE_DOWN
|
|
787
|
+
log "LANE_DOWN lane $3 is down: $1/$2 stays queued; $4"
|
|
788
|
+
finish 26 "lane down: $3 (story $1 phase $2 stays queued; $4); relaunch the $3 loop lane, then restart the driver"
|
|
789
|
+
fi
|
|
790
|
+
if [ -n "$NO_LANE" ]; then
|
|
791
|
+
# NO_LANE <id> <phase> <worker> not in the worker table <file>
|
|
792
|
+
set -- $NO_LANE
|
|
793
|
+
log "NO_LANE $3 has no row in the worker table: $1/$2 stays queued"
|
|
794
|
+
finish 27 "no lane: worker $3 has no row in the worker table (story $1 phase $2 stays queued); add the row, launch the $3 loop lane, then restart the driver"
|
|
795
|
+
fi
|
|
796
|
+
[ "$GATE" = due ] && finish 20 "regression gate due"
|
|
797
|
+
[ "$GATE" = failed ] && finish 21 "regression gate failed; if one story caused it, record: pipeline-conductor.sh gate result fail --state $STATE --story <id> --note <text>, then restart the driver"
|
|
798
|
+
[ -n "$HELD_ITEM" ] && finish 21 "story held for approval: $HELD_ITEM"
|
|
799
|
+
|
|
800
|
+
if [ "$OUT" = "" ]; then
|
|
801
|
+
act
|
|
802
|
+
if [ "$ACTED" = 0 ] && [ "$ACTIVE" = 0 ]; then
|
|
803
|
+
run_pc next --prd "$PRD" --state "$STATE"
|
|
804
|
+
if [ "$RC" = 0 ] && [ -z "$OUT" ]; then
|
|
805
|
+
run_pc report final --state "$STATE"
|
|
806
|
+
log "REPORT $OUT"
|
|
807
|
+
if [ -n "$BLOCKED_ITEMS$GO_ITEMS" ]; then
|
|
808
|
+
why="${BLOCKED_ITEMS:+ blocked by its worker: $BLOCKED_ITEMS}"
|
|
809
|
+
[ -n "$GO_ITEMS" ] && why="${why:+$why;} awaiting go: $GO_ITEMS; release with pipeline-conductor.sh go --state $STATE --story <id>"
|
|
810
|
+
finish 21 "decision needed, nothing else can move:$why"
|
|
811
|
+
fi
|
|
812
|
+
stop_lanes || finish 23 "could not confirm every loop lane stopped; see LANE_STOP_TIMEOUT"
|
|
813
|
+
finish 0 "all stories finished: $OUT"
|
|
814
|
+
fi
|
|
815
|
+
fi
|
|
816
|
+
fi
|
|
817
|
+
# wait, not a bare sleep: a TERM or INT trap runs at once instead of after the sleep
|
|
818
|
+
sleep "$INTERVAL" &
|
|
819
|
+
wait $! 2>/dev/null
|
|
820
|
+
done
|