@giamme/forge 0.11.0 → 0.12.0

package/scripts/forge-parallel.sh

@@ -18,6 +18,8 @@
 #   forge-parallel.sh split <plan-dir> <new-plan> --tasks <id,id> [--dry-run]
 #   forge-parallel.sh combine <new-plan> <plan-a> <plan-b> [<plan>...] [--verify <command>] [--dry-run]
 #   forge-parallel.sh _task <plan-dir> <task-id>     (internal; coordinator target)
+#   forge-parallel.sh _qa_only <plan-dir> <task-id>  (internal; exit 0 when the next entry only re-runs QA)
+#   forge-parallel.sh _waves <plan-dir>             (internal; rewrite waves.tsv after a live `forge control add`)
 #
 # Routing flags (plan):
 #   --dwarf <spec>          fallback for any difficulty tier not named below
@@ -430,8 +432,9 @@ EOF
 # parallelism: two dwarves editing one file produce a conflict no reviewer can
 # untangle, so they get separate waves no matter how the goal was decomposed.
 compute_waves() {
-  local PLAN="$1" T="$1/tasks.tsv" W="$1/waves.tsv"
-  : > "$W"; : > "$PLAN/deferrals.txt"
+  # Written to temp files and renamed only on success: a cycle leaves the previous files.
+  local PLAN="$1" T="$1/tasks.tsv" W="$1/.waves.tsv.$$" D="$1/.deferrals.txt.$$"
+  : > "$W"; : > "$D"
   local remaining done_ids wave n id deps files wave_files ok d progressed
   remaining="$(all_ids "$T" | tr '\n' ' ')"
   done_ids=" "; wave=0
@@ -450,7 +453,7 @@ compute_waves() {
       if [ "$ok" = 1 ] && lists_overlap "$files" "$wave_files"; then
         ok=0
         printf 'wave %s: %s deferred (file overlap with a task already in this wave)\n' \
-          "$wave" "$id" >> "$PLAN/deferrals.txt"
+          "$wave" "$id" >> "$D"
       fi
       if [ "$ok" = 1 ]; then
         printf '%s\t%s\n' "$wave" "$id" >> "$W"
@@ -460,10 +463,11 @@ compute_waves() {
         next_remaining="$next_remaining $id"
       fi
     done
-    [ "$progressed" = 1 ] || die "dependency cycle or unsatisfiable deps among:$next_remaining"
+    [ "$progressed" = 1 ] || { rm -f "$W" "$D"; die "dependency cycle or unsatisfiable deps among:$next_remaining"; }
     for id in $(awk -F'\t' -v w="$wave" '$1==w {print $2}' "$W"); do done_ids="$done_ids$id "; done
     remaining="$next_remaining"
   done
+  mv -f "$D" "$PLAN/deferrals.txt" && mv -f "$W" "$PLAN/waves.tsv"
 }
 
 render_table() {
@@ -1171,7 +1175,7 @@ do_retry() {
       --dwarf) NEWDWARF="${2:?--dwarf needs a spec}"; shift 2 ;;
       --yolo-dwarf) touch "$PLAN/yolo_dwarf"; shift ;;
       --yolo-qa)    touch "$PLAN/yolo_qa"; shift ;;
-      --verify)     printf '%s' "${2:?--verify needs a command}" > "$PLAN/verify_cmd"; shift 2 ;;
+      --verify)     printf '%s' "${2:?--verify needs a command}" > "$PLAN/verify_cmd"; rm -rf "$PLAN/verify.d"; PP_LANES_FRESH=0; shift 2 ;;
       --setup)      printf '%s' "${2:?--setup needs a command}" > "$PLAN/setup_cmd"; shift 2 ;;
       *)
         # `case` is not a loop, so `continue` here belongs to the enclosing while.
@@ -1210,6 +1214,9 @@ do_retry() {
   fi
   [ "${FORGE_RIPWIRE:-}" != off ] || touch "$PLAN/no_ripwire"
   dependencies_ready "$PLAN" "$id" || { write_results "$PLAN"; return 5; }
+  local vrepo vcheckout; vrepo="$(cat "$PLAN/repo" 2>/dev/null)"; vcheckout="$(cat "$PLAN/wt_root")/_integration"
+  [ -d "$vcheckout" ] || vcheckout="$vrepo"
+  verify_lanes_check "$PLAN" "$vrepo" "$vcheckout"
   [ ! -f "$PLAN/no_ripwire" ] || export FORGE_RIPWIRE=off
   preflight_plan "$PLAN" || return $?
   /bin/bash "$SKILL_DIR/scripts/forge-install-ripwire.sh" || true
@@ -1286,14 +1293,104 @@ verify_run_note() { # verify_run_note <out> <kind> <rc> <before> <after> <log> <
     "$(( $(date +%s) - $7 ))" "verification.run-$n.log" >> "$out/verification.runs.tsv"
 }
 
-verify_result() ( # plan, checkout, artifact directory
+# --- stoppable verification --------------------------------------------------
+# `run` sets FORGE_RUN_STOPPABLE for the verification it supervises. Then the verify command
+# runs as its own process group, and a TERM to the verification subshell stops that whole
+# group: TERM at once, KILL after a grace period. `retry` and `integrate` leave it unset and
+# keep the plain foreground command.
+verify_group_stop() { # TERM the verify command's group; KILL it after FORGE_VERIFY_GRACE seconds
+  [ -n "$FORGE_VERIFY_GROUP" ] && [ -z "$FORGE_VERIFY_TIMER" ] || return 0
+  local group="$FORGE_VERIFY_GROUP" termed=1
+  # Until the guard's setsid there is no group: the TERM goes to the guard itself, which dies
+  # of it before it starts the command. A group that forms all the same gets its TERM from
+  # the timer.
+  kill -TERM -- "-$group" 2>/dev/null || { termed=0; kill -TERM "$group" 2>/dev/null; }
+  # The timer polls in short naps rather than sleeping the whole grace in one: it ends by
+  # itself within a nap of the group's last member, and a long sleep left behind would hold
+  # the plan lock it inherited.
+  ( trap - TERM INT
+    ticks=$((FORGE_VERIFY_GRACE * 10))
+    while kill -0 -- "-$group" 2>/dev/null || { [ "$termed" = 0 ] && kill -0 "$group" 2>/dev/null; }; do
+      if [ "$termed" = 0 ] && kill -TERM -- "-$group" 2>/dev/null; then termed=1; fi
+      if [ "$ticks" -le 0 ]; then
+        if [ "$termed" = 1 ]; then kill -KILL -- "-$group" 2>/dev/null; else kill -KILL "$group" 2>/dev/null; fi
+        break
+      fi
+      sleep 0.1
+      ticks=$((ticks - 1))
+    done ) &
+  FORGE_VERIFY_TIMER=$!
+}
+
+verify_signal() { # trap handler of a stoppable verification
+  FORGE_VERIFY_STOP=1
+  verify_group_stop
+}
+
+verify_exec() { # verify_exec <checkout> <command> <log> [grace seconds, default 5] -> the command's exit code, 130 when stopped
+  local wt="$1" cmd="$2" log="$3" rc=0 group deadline
+  if [ "${FORGE_RUN_STOPPABLE:-0}" != 1 ]; then
+    (cd "$wt" && /bin/bash -c "$cmd") > "$log" 2>&1
+    return $?
+  fi
+  FORGE_VERIFY_GRACE="${4:-5}"
+  [ "$FORGE_VERIFY_STOP" = 1 ] && return 130
+  # A background job ignores SIGINT and SIGQUIT, and the command would inherit that; the
+  # wrapper restores both before the guard puts the command in a session of its own.
+  (cd "$wt" && exec python3 -c 'import os, signal, sys
+for name in (signal.SIGINT, signal.SIGQUIT):
+    signal.signal(name, signal.SIG_DFL)
+os.execv(sys.executable, [sys.executable] + sys.argv[1:])' \
+    "$SKILL_DIR/scripts/forge-guard.py" session -- /bin/bash -c "$cmd") > "$log" 2>&1 <&0 &
+  group=$!
+  FORGE_VERIFY_GROUP="$group"
+  [ "$FORGE_VERIFY_STOP" = 1 ] && verify_group_stop
+  # `wait` returns at once when a trapped signal arrives, so wait again until the job is gone.
+  wait "$group" || rc=$?
+  while kill -0 "$group" 2>/dev/null; do rc=0; wait "$group" || rc=$?; done
+  if [ "$FORGE_VERIFY_STOP" = 1 ]; then
+    # Members that outlived the leader hold the plan lock; the KILL timer ends them.
+    deadline=$((SECONDS + FORGE_VERIFY_GRACE + 5))
+    while kill -0 -- "-$group" 2>/dev/null && [ "$SECONDS" -lt "$deadline" ]; do sleep 0.1; done
+  fi
+  FORGE_VERIFY_GROUP=""
+  if [ -n "$FORGE_VERIFY_TIMER" ]; then
+    wait "$FORGE_VERIFY_TIMER" 2>/dev/null
+    FORGE_VERIFY_TIMER=""
+  fi
+  [ "$FORGE_VERIFY_STOP" = 1 ] && return 130
+  return "$rc"
+}
+
+verify_mark_interrupted() { # verify_mark_interrupted <out>
+  echo INTERRUPTED > "$1/verification.status"
+  echo 130 > "$1/verification.exit"
+  note "verification interrupted; the verify command was stopped and is not rerun"
+}
+
+# True, once the verification is marked INTERRUPTED, when a stop has arrived. Checked before
+# each run of the command (a stop that came first runs nothing and records no run) and after.
+verify_stopped() { # verify_stopped <out>
+  [ "$FORGE_VERIFY_STOP" = 1 ] || return 1
+  verify_mark_interrupted "$1"
+}
+
+verify_result() ( verify_result_body "$@" ) # plan, checkout, artifact directory
+
+# The body is a function of its own so `run` can start it as `( verify_result_body ... ) &`:
+# that one subshell is the background job, the home of the signal handler above and of
+# forge_metric_begin's EXIT trap.
+verify_result_body() { # plan, checkout, artifact directory
   local plan="$1" wt="$2" out="$3" cmd="" before after rc=0 repo jev_loaded=0 vretries
+  FORGE_VERIFY_STOP=0 FORGE_VERIFY_GROUP="" FORGE_VERIFY_TIMER="" FORGE_VERIFY_GRACE=5
+  [ "${FORGE_RUN_STOPPABLE:-0}" = 1 ] && trap verify_signal TERM INT
   mkdir -p "$out"
   forge_metric_begin "$out" verification
   forge_metric_phase verification
   repo="$(cat "$plan/repo")"
   rm -f "$out/verification.flaky" "$out/verification.retry.log" "$out/verification.coverage.txt" \
-    "$out/verification.runs.tsv" "$out"/verification.run-*.log "$out/verification.commit"
+    "$out/verification.runs.tsv" "$out"/verification.run-*.log "$out/verification.commit" \
+    "$out/verification.lanes.json" "$out"/verification.lane.*.log
   # The commit being verified, named before any early return, so every outcome (FAIL and
   # UNVERIFIED included) can be joined to the code it judged.
   git -C "$wt" rev-parse HEAD > "$out/verification.commit" 2>/dev/null || rm -f "$out/verification.commit"
@@ -1309,12 +1406,20 @@ verify_result() ( # plan, checkout, artifact directory
   if [ -f "$SKILL_DIR/scripts/forge-jev-options.sh" ]; then
     source "$SKILL_DIR/scripts/forge-jev-options.sh" 2>/dev/null && jev_loaded=1
   fi
-  if [ -s "$plan/verify_cmd" ]; then cmd="$(cat "$plan/verify_cmd")"
-  elif [ -s "$repo/.forge/verify" ]; then cmd="$(cat "$repo/.forge/verify")"; fi
+  local vsource; vsource="$(verify_source_for "$plan" "$repo")"
+  case "$vsource" in command:*) cmd="$(cat "${vsource#*$'\t'}")" ;; esac
   if [ "$(forge_tree "$wt")" != "$(git -C "$wt" rev-parse HEAD^{tree})" ]; then
     echo FAIL > "$out/verification.status"
     note "verification checkout has source changes outside its commit"; return 1
   fi
+  case "$vsource" in
+    lanes:*)
+      if [ "$jev_loaded" = 1 ] && forge_jev_active tests; then
+        note "jev: flake triage applies to a single verify command; lanes rerun their own failures"
+      fi
+      verify_lanes_run "$plan" "$wt" "$out" "${vsource#*$'\t'}" "$vretries"
+      return $? ;;
+  esac
   # Only when nothing is configured: ask Jev to name a command that actually exists
   # in the repo. Its exit code, not its stdout text, decides whether a judgment was
   # produced — any non-zero exit (no key, disabled, nothing suitable, a traceback)
@@ -1363,10 +1468,12 @@ with open(out + "/verification.jev.json", "w") as fh:
   printf '%s\n' "$cmd" > "$out/verification.command"
   before="$(forge_fingerprint "$wt")" || return 1
   local vt0; vt0="$(date +%s)"
-  (cd "$wt" && /bin/bash -c "$cmd") > "$out/verification.log" 2>&1 || rc=$?
-  echo "$rc" > "$out/verification.exit"
-  after="$(forge_fingerprint "$wt")" || return 1
+  verify_stopped "$out" && return 130
+  verify_exec "$wt" "$cmd" "$out/verification.log" || rc=$?
+  [ "$FORGE_VERIFY_STOP" = 1 ] || echo "$rc" > "$out/verification.exit"
+  after="$(forge_fingerprint "$wt")" || { verify_stopped "$out" && return 130; return 1; }
   verify_run_note "$out" initial "$rc" "$before" "$after" verification.log "$vt0"
+  verify_stopped "$out" && return 130
   echo "$before" > "$out/verification.fingerprint"
   if [ "$rc" != 0 ] || [ "$before" != "$after" ]; then
     local flaked=0 det_retried=0 n_retry=0 rc3 b3 a3
@@ -1379,9 +1486,11 @@ with open(out + "/verification.jev.json", "w") as fh:
         note "verification failed (exit $rc); rerunning to tell a flaky lane from a regression ($n_retry/$vretries)"
         b3="$(forge_fingerprint "$wt")" || return 1
         vt0="$(date +%s)"
-        (cd "$wt" && /bin/bash -c "$cmd") > "$out/verification.retry.log" 2>&1 || rc3=$?
-        a3="$(forge_fingerprint "$wt")" || return 1
+        verify_stopped "$out" && return 130
+        verify_exec "$wt" "$cmd" "$out/verification.retry.log" || rc3=$?
+        a3="$(forge_fingerprint "$wt")" || { verify_stopped "$out" && return 130; return 1; }
         verify_run_note "$out" rerun "$rc3" "$b3" "$a3" verification.retry.log "$vt0"
+        verify_stopped "$out" && return 130
         if [ "$rc3" = 0 ] && [ "$b3" = "$a3" ]; then
           flaked=1
           {
@@ -1446,10 +1555,12 @@ print(v if v is not None else "0.90")
           note "jev: known_flake (confidence $confidence >= $flake_act), corroborated by a recorded trap — retrying once"
           before2="$(forge_fingerprint "$wt")" || return 1
           vt0="$(date +%s)"
-          (cd "$wt" && /bin/bash -c "$cmd") > "$out/verification.retry.log" 2>&1 || rc2=$?
+          verify_stopped "$out" && return 130
+          verify_exec "$wt" "$cmd" "$out/verification.retry.log" || rc2=$?
           rc2="${rc2:-0}"
-          after2="$(forge_fingerprint "$wt")" || return 1
+          after2="$(forge_fingerprint "$wt")" || { verify_stopped "$out" && return 130; return 1; }
           verify_run_note "$out" jev_retry "$rc2" "$before2" "$after2" verification.retry.log "$vt0"
+          verify_stopped "$out" && return 130
           if [ "$rc2" = 0 ] && [ "$before2" = "$after2" ]; then
             flaked=1
             note "jev: retry passed — treating as a corroborated flake, not a regression"
@@ -1490,7 +1601,7 @@ with open(path, "w") as fh:
     return 1
   fi
   echo PASS > "$out/verification.status"
-)
+}
 
 write_results() { # write_results <plan>
   local PLAN="$1" T="$1/tasks.tsv" run_id id
@@ -1511,7 +1622,29 @@ forecast_report() {
 }
 
 # --- run ----------------------------------------------------------------------
-do_run() (
+# Waits for a background job and, when a trapped signal ends the wait early, waits again
+# until the job is gone; its exit code is left in run_rc.
+run_wait() { # run_wait <pid>
+  run_rc=0
+  wait "$1" || run_rc=$?
+  while kill -0 "$1" 2>/dev/null; do run_rc=0; wait "$1" || run_rc=$?; done
+}
+
+# The handler of do_run's subshell: a signal reaches the job in progress (the scheduler,
+# then the verification) as TERM. Before any job exists there is nothing to clean up.
+run_signal() {
+  run_stop=1
+  if [ -n "$run_child" ]; then kill -TERM "$run_child" 2>/dev/null
+  elif [ "$run_phase" = pre ]; then exit 130; fi
+}
+
+# Not a ( ) function: main runs it as `( do_run ... ) &`, one subshell that is both the
+# background job and the home of its signal handler. A function called with `&` would be a
+# second subshell level, and an async function does not run its EXIT trap.
+do_run() {
+  local run_child="" run_stop=0 run_phase=pre run_rc=0 interrupted=0 vstopped=0
+  FORGE_RUN_STOPPABLE=1
+  trap run_signal TERM INT
   local PLAN="${1:?run needs a plan dir}"; shift
   local fractal_pipeline_args=("$@")
   local MAXP=3 DRY=0 jev_rc=0 FORECAST=1
@@ -1527,7 +1660,7 @@ do_run() (
       --max-parallel) MAXP="${2:?}"; shift 2 ;;
       --yolo-dwarf)   touch "$PLAN/yolo_dwarf"; shift ;;
       --yolo-qa)      touch "$PLAN/yolo_qa"; shift ;;
-      --verify)     printf '%s' "${2:?--verify needs a command}" > "$PLAN/verify_cmd"; shift 2 ;;
+      --verify)     printf '%s' "${2:?--verify needs a command}" > "$PLAN/verify_cmd"; rm -rf "$PLAN/verify.d"; PP_LANES_FRESH=0; shift 2 ;;
       --setup)        printf '%s' "${2:?--setup needs a command}" > "$PLAN/setup_cmd"; shift 2 ;;
       --dry-run)      DRY=1; shift ;;
       *)
@@ -1623,22 +1756,46 @@ do_run() (
   # Also before any model is paid for: what this plan is likely to cost, and which tasks
   # history says may bounce. Advisory, budgeted and silent without history; it never stops a run.
   [ "$FORECAST" = 1 ] && forecast_report "$PLAN"
+  # Invalid verification lanes stop the run here, before any model is paid for.
+  verify_lanes_check "$PLAN" "$REPO" "$int_wt"
   # Say now, before any model is paid for, which suites the verify command will not run.
   verify_coverage_report "$REPO" "$(verify_command_for "$PLAN" "$REPO")" "$PLAN/verification.coverage.txt"
   local sched_rc=0 paused=0 sched_args=()
   [ "$PP_RETRY_FAILED" -gt 0 ] && sched_args+=(--retry-failed "$PP_RETRY_FAILED")
   [ "$PP_INFRA_RETRIES" -gt 0 ] && sched_args+=(--infra-retries "$PP_INFRA_RETRIES")
   forge_metric_phase dispatch
-  python3 "$SKILL_DIR/scripts/forge-schedule.py" "$SELF" "$PLAN" "$MAXP" ${sched_args[@]+"${sched_args[@]}"} || sched_rc=$?
+  # In the background so a signal to this shell ends the wait; the scheduler gets it as TERM,
+  # stops its tasks and returns 130.
+  run_phase=scheduler
+  python3 "$SKILL_DIR/scripts/forge-schedule.py" "$SELF" "$PLAN" "$MAXP" ${sched_args[@]+"${sched_args[@]}"} <&0 &
+  run_child=$!
+  [ "$run_stop" = 1 ] && kill -TERM "$run_child" 2>/dev/null
+  run_wait "$run_child"; sched_rc=$run_rc
+  run_child=""
   [ "$sched_rc" != 0 ] && failed=1
   [ "$sched_rc" = 8 ] && paused=1
 
   forge_metric_phase verification
+  # Counted after that phase call: a signal that arrived while it ran is handled only once it
+  # returned, and must still keep verification from starting.
+  [ "$sched_rc" = 130 ] || [ "$run_stop" = 1 ] && interrupted=1
   # An infrastructure pause leaves the run unfinished, so there is nothing yet worth
-  # verifying; the next `run` resumes and verifies at its end.
-  if [ "$paused" = 0 ]; then
-    verify_result "$PLAN" "$int_wt" "$PLAN" || failed=1
+  # verifying; the next `run` resumes and verifies at its end. An interrupted run has
+  # merged only part of its work: a verification of that is not worth its minutes, and it
+  # would run again as the rerun if it were killed.
+  if [ "$paused" = 0 ] && [ "$interrupted" = 0 ]; then
+    run_phase=verification
+    ( verify_result_body "$PLAN" "$int_wt" "$PLAN" ) <&0 &
+    run_child=$!
+    [ "$run_stop" = 1 ] && kill -TERM "$run_child" 2>/dev/null
+    run_wait "$run_child"
+    [ "$run_rc" != 0 ] && failed=1
+    [ "$run_rc" = 130 ] && vstopped=1
+    run_child=""
+    # A stop that killed the subshell before its handler was set leaves no record: it counts too.
+    [ "$vstopped" = 1 ] || [ "$run_stop" = 1 ] && interrupted=1
   fi
+  run_phase=results
   write_results "$PLAN"
   # Only prune worktrees for work that is safely merged, and only after the record
   # of it has been written and read back. A failed task keeps its worktree and
@@ -1670,15 +1827,27 @@ do_run() (
   done
   [ "$drift" = 1 ] && echo
 
+  # From this run's own flags, never from a verification.status an earlier run left.
+  if [ "$interrupted" = 1 ]; then
+    if [ "$vstopped" = 1 ]; then echo "interrupted during verification, which was stopped and not rerun."
+    else echo "interrupted; verification was skipped."; fi
+  fi
+  local stopped_ids; stopped_ids="$(awk -F'\t' '$2=="INTERRUPTED" {print $1}' "$PLAN/results.tsv")"
+  if [ -n "$stopped_ids" ]; then
+    echo "A later run does not dispatch an interrupted task; continue each one with:"
+    for id in $stopped_ids; do echo "  forge-parallel.sh retry $PLAN $id"; done
+  fi
+  [ "$interrupted" = 1 ] || [ -n "$stopped_ids" ] && echo
   # The retry hint is for tasks that actually failed; a run that only failed verification,
-  # or only paused on infrastructure, has nothing to retry task by task.
-  if [ "$(awk -F'\t' '$2!="MERGED" && $2!="INFRA" && $2!="PENDING" && $2!="BLOCKED" {n++} END {print n+0}' "$PLAN/results.tsv")" -gt 0 ]; then
+  # or only paused on infrastructure, has nothing to retry task by task. An interrupted task
+  # has its own hint above.
+  if [ "$(awk -F'\t' '$2!="MERGED" && $2!="INFRA" && $2!="PENDING" && $2!="BLOCKED" && $2!="INTERRUPTED" {n++} END {print n+0}' "$PLAN/results.tsv")" -gt 0 ]; then
     echo "retry a failed task with its reviewer's findings:"
     echo "  forge-parallel.sh retry $PLAN <task-id> [--dwarf <spec>]"
     echo "or let a run retry them itself:  forge-parallel.sh run $PLAN --retry-failed 1"
     echo
   fi
-  if [ "$failed" = 1 ] && [ "$paused" = 0 ] && [ "$(cat "$PLAN/verification.status" 2>/dev/null)" = FAIL ]; then
+  if [ "$failed" = 1 ] && [ "$paused" = 0 ] && [ "$interrupted" = 0 ] && [ "$(cat "$PLAN/verification.status" 2>/dev/null)" = FAIL ]; then
     echo "combined verification failed: see $PLAN/verification.log"
     echo
   fi
@@ -1695,10 +1864,11 @@ do_run() (
   else
     echo "Your branch and working tree were not touched."
   fi
+  [ "$interrupted" = 1 ] && return 130
   [ "$paused" = 1 ] && return 8
   [ "$failed" = 1 ] && return 5
   return 0
-)
+}
 
 # --- integrate ----------------------------------------------------------------
 do_integrate() {
@@ -1789,7 +1959,19 @@ esac
 need_fn() { declare -F "$1" >/dev/null || die "'$CMD' is not available: $1 is missing" 3; }
 case "$CMD" in
   plan)      do_plan "$@"; rc=$?; forge_update_notice; exit "$rc" ;;
-  run)       do_run "$@"; rc=$?; forge_update_notice; exit "$rc" ;;
+  run)
+    # The signal handler lives here, at the pid the user started: do_run is a subshell, and
+    # bash runs a trap only between commands, so it is started in the background and waited
+    # for. A signal goes to it as TERM; do_run stops the scheduler or the verification.
+    run_signalled=0 run_job=""
+    trap 'run_signalled=1; [ -n "$run_job" ] && kill -TERM "$run_job" 2>/dev/null' TERM INT
+    ( do_run "$@" ) <&0 &
+    run_job=$!
+    [ "$run_signalled" = 1 ] && kill -TERM "$run_job" 2>/dev/null
+    run_wait "$run_job"; rc=$run_rc
+    trap - TERM INT
+    [ "$run_signalled" = 1 ] && [ "$rc" -gt 128 ] && rc=130
+    forge_update_notice; exit "$rc" ;;
   retry)     do_retry "$@" ;;
   integrate) do_integrate "$@" ;;
   accept)    need_fn do_accept;  do_accept "$@" ;;
@@ -1799,5 +1981,7 @@ case "$CMD" in
   _task)     do_task "$@" ;;
   _merge)    merge_task "$@" ;;
   _prepare_retry) retry_for_scheduler "$@" ;;
+  _qa_only)  qa_only_next "$@" ;;
+  _waves)    compute_waves "$@" ;;
   *) die "unknown subcommand '$CMD'" ;;
 esac

package/scripts/forge-ripwire.py

@@ -52,7 +52,7 @@ def bounded(command, cwd, deadline, limit=16384):
         # Also kill descendants that closed their pipes before the parent exited.
         try:
             os.killpg(process.pid, signal.SIGKILL)
-        except ProcessLookupError:
+        except (ProcessLookupError, PermissionError):   # macOS: EPERM for a group of zombies only
             pass
         process.wait()
         streams.close()

package/scripts/forge-route.py

@@ -20,9 +20,15 @@ bias (inverse-propensity weighting needs the probability each choice had).
       The retry policy: after <plan>/escalate_after (default 3; 0 = off) failed attempts on
       one spec, the next attempt goes to the next tier up of the user's pools instead of
       repeating the same dispatch. Prints that spec when one is due. When none exists, writes
-      tasks/<id>/escalation.stop and split.suggested and exits 3. `--check` (run after every
-      attempt) also records a due escalation in escalation.next, and clears the markers once
-      the streak no longer holds. A pending tasks/<id>/escalation.pin (a human's one-shot
+      tasks/<id>/escalation.stop and split.suggested and exits 3. When <plan>/escalate_effort
+      contains `on` and no higher tier has another model, the same alias and harness is tried
+      once more at the next effort its pairing accepts (weakest above the current one; a spec
+      that names no effort counts as medium, a dwarf dispatch's default) before that stop.
+      The streak then counts per effort, so the new effort starts again. At the pairing's
+      highest effort the task still stops, and split.suggested says that effort was tried too.
+      With the file absent the streak and the stop are exactly as before. `--check` (run after
+      every attempt) also records a due escalation in escalation.next, and clears the markers
+      once the streak no longer holds. A pending tasks/<id>/escalation.pin (a human's one-shot
       choice, applied by prepare_retry) is what `--check` reports as the next model instead
       (escalation.next = its spec), whatever escalate_after says, unless the next retry is a
       QA-only resume or the plan is Fractal (pins never apply there). `--qa-only-next` is
@@ -60,6 +66,7 @@ import os
 import random
 import secrets
 import stat
+import subprocess
 import sys
 import tempfile
 import time
@@ -71,7 +78,12 @@ REGISTRY = SCRIPTS.parent / 'registry.tsv'
 FAILED = ('FAIL', 'TIMEOUT')                       # a judgment against the spec that ran
 NEUTRAL = ('INFRA', 'UNKNOWN', 'ERROR', 'RUNNING', 'INTERRUPTED')   # say nothing about it
 DEFAULT_ESCALATE_AFTER = 3
+DEFAULT_EFFORT = 'medium'                         # forge-dispatch.sh dwarf default
 LOCK_WAIT = 5.0                                    # seconds to wait for P/control.lock
+# forge-dispatch.sh rank_of. Only for "which accepted effort is above this one"; the set of
+# efforts a pairing accepts comes from `forge-dispatch.sh efforts`, not from this table.
+# ultra and ultracode share a rank: each is its own harness's top, under different names.
+EFFORT_RANK = {'low': 0, 'medium': 1, 'high': 2, 'xhigh': 3, 'max': 4, 'ultra': 5, 'ultracode': 5}
 
 
 def rng_for(task: str):
@@ -209,6 +221,64 @@ def stronger(spec: str, pools: dict, registry: dict):
     return None, 'it is already in the highest tier of this plan\'s pools (%s)' % TIERS[rank]
 
 
+# --- capacity fallback ---------------------------------------------------------------
+
+def pair_of(spec: str, registry: dict):
+    """(harness, model id) a spec runs on, as forge-dispatch.sh's lookup resolves it; None when it cannot."""
+    parts = str(spec).split(':')
+    alias, harness = parts[0], ':'.join(parts[2:])
+    rows = registry.get(alias)
+    if rows:
+        if not harness:
+            return next(iter(rows.items()))
+        return harness, rows.get(harness, alias)
+    return (harness, alias) if harness else None
+
+
+def parse_excluded(text: str) -> set:
+    """`harness/model` items, comma-separated; `harness/*` excludes every model on that harness."""
+    out = set()
+    for item in (text or '').split(','):
+        harness, slash, model = item.strip().partition('/')
+        if harness and slash and model:
+            out.add((harness, model))
+    return out
+
+
+def fallback_pick(current: str, pools: dict, registry: dict, excluded: set):
+    """The next member of the current dwarf's own tier pool, in pool order and wrapping around,
+    that runs another model on a pair nobody paused; None when there is none."""
+    rank = rank_of(current, pools, registry)
+    if rank is None:
+        return None
+    pool = pools[TIERS[rank]]
+    own = canonical(current, registry)
+    keys = [canonical(s, registry) for s in pool]
+    start = keys.index(own) if own in keys else -1
+    for step in range(1, len(pool) + 1):
+        candidate = pool[(start + step) % len(pool)]
+        pair = pair_of(candidate, registry)
+        if canonical(candidate, registry) == own or pair is None:
+            continue
+        if pair in excluded or (pair[0], '*') in excluded:
+            continue
+        return candidate
+    return None
+
+
+def fallback(args) -> int:
+    plan = Path(os.path.realpath(args.plan))
+    current = _dwarf_of(plan, args.task)
+    if not current or current in ('-', 'UNASSIGNED'):
+        return 1
+    pick = fallback_pick(current, load_pools(plan / 'fractal-routing.json'), registry_models(),
+                         parse_excluded(args.exclude))
+    if not pick:
+        return 1
+    print(pick)
+    return 0
+
+
 # --- exploration -------------------------------------------------------------------
 
 def explore(args) -> str:
@@ -250,20 +320,87 @@ def _dwarf_of(plan: Path, task: str) -> str:
     return ''
 
 
-def failed_streak(rows: list, current: str, registry: dict) -> int:
+def _spec_parts(spec: str):
+    """(alias, effort, harness) of alias[:effort[:harness]]. Effort and harness are '' when unnamed."""
+    alias, sep, rest = str(spec).partition(':')
+    if not sep:
+        return alias, '', ''
+    effort, sep, harness = rest.partition(':')
+    return alias, effort, harness if sep else ''
+
+
+def _effort_of(spec: str) -> str:
+    """The effort a spec names, or medium when it names none (what a dwarf dispatch runs)."""
+    _alias, effort, _harness = _spec_parts(spec)
+    return effort or DEFAULT_EFFORT
+
+
+def _effort_step(plan: Path) -> bool:
+    try:
+        return (plan / 'escalate_effort').read_text().strip() == 'on'
+    except OSError:
+        return False
+
+
+def _accepted_efforts(spec: str) -> list:
+    """Efforts `spec`'s pairing accepts, weakest first. Empty when dispatch cannot say."""
+    try:
+        done = subprocess.run(
+            ['bash', str(SCRIPTS / 'forge-dispatch.sh'), 'efforts', '--spec', spec],
+            capture_output=True, text=True, timeout=30)
+    except (OSError, subprocess.SubprocessError):
+        return []
+    if done.returncode != 0:
+        return []
+    return [line.strip() for line in done.stdout.splitlines() if line.strip()]
+
+
+def _effective_effort(requested: str, accepted: list):
+    """The effort a dwarf dispatch runs when `requested` is asked of a pairing accepting
+    `accepted`: as asked when accepted, else the strongest accepted at or below it, else the
+    weakest accepted (forge-dispatch.sh's clamp)."""
+    if requested in accepted:
+        return requested
+    want = EFFORT_RANK.get(requested)
+    below = [e for e in accepted if want is not None and EFFORT_RANK.get(e, 99) <= want]
+    return below[-1] if below else (accepted[0] if accepted else None)
+
+
+def _next_effort_spec(spec: str):
+    """The same alias and harness at the next accepted effort above the one a dispatch of
+    `spec` actually runs, or None."""
+    alias, _effort, harness = _spec_parts(spec)
+    accepted = _accepted_efforts(spec)
+    rank = EFFORT_RANK.get(_effective_effort(_effort_of(spec), accepted) or '')
+    if not alias or rank is None:
+        return None
+    for effort in accepted:
+        other = EFFORT_RANK.get(effort)
+        if other is not None and other > rank:
+            return '%s:%s:%s' % (alias, effort, harness) if harness else '%s:%s' % (alias, effort)
+    return None
+
+
+def failed_streak(rows: list, current: str, registry: dict, by_effort: bool = False) -> int:
     """Distinct trailing attempt numbers that failed on `current`'s model.
 
     rows: (attempt, status, spec-or-None) oldest first. A row without a spec predates the
     column and is taken as the current spec. Infra stops, missing verdicts and forge errors
     judge nothing and are skipped; a pass, a merge or another model ends the streak.
+    With by_effort, another effort of the same model ends it too (a bump starts a new
+    streak). A spec that names no effort counts as medium. With by_effort off the count
+    is by model only, as before.
     """
     model = canonical(current, registry)
+    effort = _effort_of(current) if by_effort else None
     seen = set()
     for attempt, status, spec in reversed(rows):
         if status in NEUTRAL:
             continue
         if spec and canonical(spec, registry) != model:
             break
+        if effort is not None and _effort_of(spec or current) != effort:
+            break
         if status not in FAILED:
             break
         seen.add(attempt)
@@ -439,13 +576,19 @@ def _decide(plan: Path, tdir: Path, args, after: int):
     if not current or current in ('-', 'UNASSIGNED'):
         return (), (), '', 0
     registry = registry_models()
-    streak = failed_streak(attempt_rows(tdir), current, registry)
+    step = _effort_step(plan)
+    streak = failed_streak(attempt_rows(tdir), current, registry, by_effort=step)
     if streak < after:
         return MARKERS, (), '', 0
     target, why = stronger(current, load_pools(plan / 'fractal-routing.json'), registry)
+    # Only once no higher tier has a model. A tier-up is still the stronger move.
+    if target is None and step:
+        target = _next_effort_spec(current)
     if target:
         writes = (('escalation.next', target + '\n'),) if args.check else ()
         return ('escalation.stop', 'split.suggested'), writes, target, 0
+    if step:
+        why = '%s. The highest effort this pairing accepts was tried as well' % why
     split = ('%s failed %d attempts on %s, and %s. Another attempt on the same model rarely passes, '
              'so split the task into smaller ones, or retry it with an explicit --dwarf <spec>.\n'
              % (args.task, streak, current, why))
@@ -536,6 +679,10 @@ def main(argv=None) -> int:
     p.add_argument('--check', action='store_true')
     p.add_argument('--qa-only-next', choices=('yes', 'no'), default=None,
                    help='whether the next entry only re-runs QA, as the caller measured it')
+    p = sub.add_parser('fallback', help='the same-tier model a capacity stop moves to')
+    p.add_argument('--plan', required=True)
+    p.add_argument('--task', required=True)
+    p.add_argument('--exclude', default='', help='harness/model pairs to skip, comma-separated (harness/* for all)')
     p = sub.add_parser('record', help=argparse.SUPPRESS)
     p.add_argument('--plan', required=True)
     p.add_argument('--task', required=True)
@@ -551,6 +698,11 @@ def main(argv=None) -> int:
         return 0
     if args.command == 'record':
         return record_escalation(args)
+    if args.command == 'fallback':
+        try:
+            return fallback(args)
+        except Exception:
+            return 1
     try:
         return escalation(args)
     except Exception: