@giamme/forge 0.11.1 → 0.12.0

package/scripts/forge-guard.py

@@ -17,6 +17,19 @@
       bounded ~2s, KILL, print how many were found. Never exits nonzero for an ordinary
       outcome, and never touches its own or its caller's group.
 
+  forge-guard.py identity <pid>
+      Print the process's start time, which survives exec. Dispatch records it with the
+      harness pid right after `$!`, when that pid may still be the subshell or this helper
+      before the exec into the harness, so the command name cannot identify it.
+
+  forge-guard.py stop <record-file>
+      The backstop for a harness its dispatch could not stop. The record is
+      `<pid><TAB><identity>`. A live pid with the recorded start time gets its group
+      signalled (or itself alone, before setsid made it a leader): TERM, ~2s, KILL. A live
+      pid with another start time was reused and is left alone. A dead pid's group may
+      still hold the harness's background jobs, and cannot have been reused while they
+      live, so it is reaped. The record is removed; prints how many processes were signalled.
+
   forge-guard.py promises <file> [--tail-bytes N]
       Scan the tail of an implementer's final message for promises of later work ("it is
       still running in the background", "I'll check back"). One `pattern<TAB>excerpt`
@@ -115,39 +128,153 @@ def wait_gone(pgid: int, seconds: float) -> bool:
     return (not members) if members is not None else not group_exists(pgid)
 
 
-def reap(pgid: int, orphans_file: str) -> int:
-    # Never signal ourselves, the dispatcher that called us, or init.
+def _forbidden() -> set:
+    # Never signal ourselves, the process that called us, or init.
     forbidden = {0, 1, os.getpgrp()}
     try:
         forbidden.add(os.getpgid(os.getppid()))
     except OSError:
         pass
-    if pgid in forbidden or pgid < 0:
-        print(0)
+    return forbidden
+
+
+def _reap_group(pgid: int, orphans_file: str | None) -> int:
+    """TERM, then KILL, whatever is alive in the group; how many were found."""
+    if pgid in _forbidden() or pgid < 0:
         return 0
     if not group_exists(pgid):
-        print(0)
         return 0
     members = list_group(pgid)
     if members is not None and not members:
         # Only zombies (or nothing visible) left; nothing worth reporting or killing.
-        print(0)
         return 0
     found = members if members is not None else [(0, '(unknown: ps unavailable; process group %d still has live members)' % pgid)]
-    try:
-        with open(orphans_file, 'w') as handle:
-            for pid, command in found:
-                handle.write('%s\t%s\n' % (pid or '?', ' '.join(command.split()) or '?'))
-    except OSError:
-        pass
+    if orphans_file:
+        try:
+            with open(orphans_file, 'w') as handle:
+                for pid, command in found:
+                    handle.write('%s\t%s\n' % (pid or '?', ' '.join(command.split()) or '?'))
+        except OSError:
+            pass
     for sig, wait in ((signal.SIGTERM, TERM_WAIT_S), (signal.SIGKILL, KILL_WAIT_S)):
         try:
             os.killpg(pgid, sig)
-        except (ProcessLookupError, PermissionError, OSError):
+        except OSError:
             break
         if wait_gone(pgid, wait):
             break
-    print(len(found))
+    return len(found)
+
+
+def reap(pgid: int, orphans_file: str) -> int:
+    print(_reap_group(pgid, orphans_file))
+    return 0
+
+
+# --- identity / stop ----------------------------------------------------------
+def _ps_field(pid: int, field: str) -> str | None:
+    """One ps column for one pid, whitespace-normalised; None when the pid is gone or ps
+    is unavailable. LC_ALL=C keeps lstart's format the same for every caller."""
+    if pid <= 0:
+        return None
+    for ps in PS_CANDIDATES:
+        if not os.access(ps, os.X_OK):
+            continue
+        try:
+            out = subprocess.run([ps, '-p', str(pid), '-o', field + '='], stdout=subprocess.PIPE,
+                                 stderr=subprocess.DEVNULL, timeout=5,
+                                 env=dict(os.environ, LC_ALL='C')).stdout.decode(errors='replace')
+        except (OSError, subprocess.SubprocessError):
+            continue
+        return ' '.join(out.split()) or None
+    return None
+
+
+def identity(pid: int) -> str | None:
+    return _ps_field(pid, 'lstart')
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return False
+    # A zombie still answers kill(0) until its parent waits for it.
+    return not (_ps_field(pid, 'stat') or 'Z').startswith('Z')
+
+
+def _wait_pid_gone(pid: int, seconds: float) -> bool:
+    deadline = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        if not _pid_alive(pid):
+            return True
+        time.sleep(0.05)
+    return not _pid_alive(pid)
+
+
+def _stop_live(pid: int) -> int:
+    try:
+        leader = os.getpgid(pid) == pid
+    except OSError:
+        leader = False
+    if leader:
+        if pid in _forbidden():
+            return 0
+        members = list_group(pid)
+        count = len(members) if members else 1
+        for sig, wait in ((signal.SIGTERM, TERM_WAIT_S), (signal.SIGKILL, KILL_WAIT_S)):
+            try:
+                os.killpg(pid, sig)
+            except OSError:
+                break
+            if wait_gone(pid, wait):
+                break
+        return count
+    # Recorded before setsid: the pid is still in its dispatcher's group, so only it is ours.
+    for sig, wait in ((signal.SIGTERM, TERM_WAIT_S), (signal.SIGKILL, KILL_WAIT_S)):
+        try:
+            os.kill(pid, sig)
+        except OSError:
+            break
+        if _wait_pid_gone(pid, wait):
+            break
+    return 1
+
+
+def stop(record: str) -> int:
+    try:
+        with open(record) as handle:
+            line = handle.readline()
+    except FileNotFoundError:
+        print(0)
+        return 0
+    except OSError as error:
+        print('forge-guard: cannot read %s: %s' % (record, error), file=sys.stderr)
+        print(0)
+        return 0
+    pid_text, _, recorded = line.rstrip('\n').partition('\t')
+    try:
+        pid = int(pid_text)
+    except ValueError:
+        pid = 0
+    recorded = ' '.join(recorded.split())
+    count = 0
+    if pid > 1:
+        if _pid_alive(pid):
+            # A mismatch, or no way to tell, means the pid now belongs to someone else.
+            if recorded and identity(pid) == recorded:
+                count = _stop_live(pid)
+        else:
+            count = _reap_group(pid, None)
+    try:
+        os.unlink(record)
+    except OSError:
+        pass
+    print(count)
     return 0
 
 
@@ -229,7 +356,7 @@ def promises(path: str, tail_bytes: int = 3000) -> int:
 def main(argv: list[str]) -> int:
     if not argv:
         print('usage: forge-guard.py ping | session -- CMD... | reap <pgid> <orphans-file> | '
-              'promises <file> [--tail-bytes N]', file=sys.stderr)
+              'identity <pid> | stop <record-file> | promises <file> [--tail-bytes N]', file=sys.stderr)
         return 2
     command, rest = argv[0], argv[1:]
     try:
@@ -245,6 +372,20 @@ def main(argv: list[str]) -> int:
                 print('usage: forge-guard.py reap <pgid> <orphans-file>', file=sys.stderr)
                 return 2
             return reap(int(rest[0]), rest[1])
+        if command == 'identity':
+            if len(rest) != 1:
+                print('usage: forge-guard.py identity <pid>', file=sys.stderr)
+                return 2
+            found = identity(int(rest[0]))
+            if found is None:
+                return 1
+            print(found)
+            return 0
+        if command == 'stop':
+            if len(rest) != 1:
+                print('usage: forge-guard.py stop <record-file>', file=sys.stderr)
+                return 2
+            return stop(rest[0])
         if command == 'promises':
             tail = 3000
             if '--tail-bytes' in rest:

package/scripts/forge-lib-plans.sh

@@ -130,7 +130,7 @@ plans_make_integration() { # plans_make_integration <repo> <branch> <commit> <wt
 
 # Per-plan option files carried to a derived plan. The cost-bearing choices a user made
 # for this plan apply to the plan made from it; `planner` does not (nothing plans again).
-PLANS_OPTION_FILES="no_ripwire no_memory yolo_dwarf yolo_qa reviewer_session qa_threshold timeout verify_retries verify_cmd setup_cmd final_qa shadow_qa shadow_rate explore escalate_after fractal-routing.json fractal-selection.json jev-selection.json"
+PLANS_OPTION_FILES="no_ripwire no_memory yolo_dwarf yolo_qa reviewer_session qa_threshold timeout verify_retries verify_cmd verify_parallel setup_cmd final_qa shadow_qa shadow_rate explore escalate_after fractal-routing.json fractal-selection.json jev-selection.json"
 
 # --- accept ---------------------------------------------------------------------
 # A reviewer's FAIL is the QA gate; overriding it is the user's call and never forge's or
@@ -524,7 +524,7 @@ Retry or accept them (or split them off into their own plan) first." 3
   # and what the dataset records; readers only test that the file exists.
   all=1; for p in "${SRC[@]}"; do [ -f "$p/reviewer_session" ] || all=0; done
   [ "$all" = 1 ] && printf '%s\n' on > "$NEW/reviewer_session"
-  for f in qa_threshold timeout verify_retries; do
+  for f in qa_threshold timeout verify_retries verify_parallel; do
     same=1
     for p in "${SRC[@]}"; do
       if [ ! -f "$p/$f" ] || ! cmp -s "$p/$f" "${SRC[0]}/$f"; then same=0; fi
@@ -541,7 +541,11 @@ Retry or accept them (or split them off into their own plan) first." 3
   for p in "${SRC[@]}"; do [ -s "$p/setup_cmd" ] && { cp -p "$p/setup_cmd" "$NEW/setup_cmd"; break; }; done
   if [ -n "$VERIFY" ]; then printf '%s' "$VERIFY" > "$NEW/verify_cmd"
   else
-    for p in "${SRC[@]}"; do [ -s "$p/verify_cmd" ] && { cp -p "$p/verify_cmd" "$NEW/verify_cmd"; break; }; done
+    # The first source that declares verification gives it: its lanes, else its command.
+    for p in "${SRC[@]}"; do
+      if verify_lanes_present "$p/verify.d"; then cp -Rp "$p/verify.d" "$NEW/verify.d"; break; fi
+      [ -s "$p/verify_cmd" ] && { cp -p "$p/verify_cmd" "$NEW/verify_cmd"; break; }
+    done
   fi
   for f in fractal-routing.json jev-selection.json; do
     for p in "${SRC[@]}"; do [ -f "$p/$f" ] && { cp -p "$p/$f" "$NEW/$f"; break; }; done
@@ -743,6 +747,8 @@ except Exception:
   for f in $PLANS_OPTION_FILES; do
     [ ! -e "$PLAN/$f" ] || cp -p "$PLAN/$f" "$NEW/$f" || ok=0
   done
+  # The plan's verification lanes travel with it, like its --verify.
+  [ ! -d "$PLAN/verify.d" ] || cp -Rp "$PLAN/verify.d" "$NEW/verify.d" || ok=0
   for id in $(all_ids "$T"); do
     case "$SEL" in *" $id "*) ;; *) continue ;; esac
     plans_copy_taskdir "$PLAN/tasks/$id" "$NEW/tasks/$id" || ok=0

package/scripts/forge-lib-review.sh

@@ -94,7 +94,7 @@ review_warn_unmerged() { # review_warn_unmerged <plan>
 #   notes  what forge observed that no per-task reviewer was shown
 review_context() { # review_context <pre|notes> <plan> <tasks.summary> <verification-dir>
   python3 - "$1" "$2" "$3" "$4" "$REVIEW_REQ_BUDGET" <<'PY'
-import os, re, sys
+import json, os, re, sys
 
 mode, plan, summary, vdir, budget = sys.argv[1:6]
 BUDGET = int(budget)
@@ -195,13 +195,32 @@ else:
              'A promise of later work means a check may never have run.')
     ver = []
     status = read(os.path.join(vdir, 'verification.status'))
+    lanes = None
+    try:
+        doc = json.loads(read(os.path.join(vdir, 'verification.lanes.json')) or b'null')
+        lanes = [l for l in doc.get('lanes') or [] if isinstance(l, dict) and l.get('name')]
+    except (ValueError, AttributeError):
+        pass
     if status and status.strip():
         ver.append('status: %s' % text(status))
         cmd = read(os.path.join(vdir, 'verification.command'))
-        if cmd and cmd.strip():
+        if lanes is not None:
+            # Lanes: each one's state; a flaky lane passed only on its rerun, a known lane failed
+            # only tests the project's baseline lists.
+            ver.append('lanes (each ran on its own; a failed lane was rerun alone):')
+            for l in lanes:
+                ver.append('  %s: %s' % (l['name'], l.get('state') or 'unknown'))
+            by = lambda state: [l['name'] for l in lanes if l.get('state') == state]
+            if by('fail'):
+                ver.append('failed: ' + ', '.join('lane ' + n for n in by('fail')))
+            for n in by('flaky'):
+                ver.append('FLAKY: lane %s failed once and passed on rerun, so its pass is not clean' % n)
+            for n in by('known'):
+                ver.append('known failures only: lane %s (every failing test is in its baseline)' % n)
+        elif cmd and cmd.strip():
             ver.append('command: %s' % text(cmd))
     flaky = read(os.path.join(vdir, 'verification.flaky'))
-    if flaky and flaky.strip():
+    if flaky and flaky.strip() and lanes is None:
         ver.append('FLAKY: the command failed once and passed on rerun, so the pass is not clean:')
         ver += ['  ' + l for l in text(flaky).splitlines() if not l.startswith(('first_log=', 'retry_log='))]
     cover = read(os.path.join(vdir, 'verification.coverage.txt'))

package/scripts/forge-lib-task.sh

@@ -406,15 +406,43 @@ prepare_resume() { # prepare_resume <plan> <id>
   return 0
 }
 
+# --infra-fallback: a dwarf-stage capacity stop moves the task to the next healthy member of
+# its own tier pool (forge-route.py fallback), recorded like an escalation with source
+# `fallback`. No attempt is spent: the stop already refunded it. Returns 1, changing nothing,
+# when there is no candidate, a pin is waiting (a human's choice wins) or the plan is Fractal.
+prepare_fallback() { # prepare_fallback <plan> <id> <excluded harness/model,...>
+  local PLAN="$1" id="$2" tdir="$1/tasks/$2" cur pick
+  [ -e "$tdir/escalation.pin" ] && return 1
+  forge_plan_is_fractal "$PLAN" && return 1
+  cur="$(field "$PLAN/tasks.tsv" "$id" dwarf)"
+  pick="$(python3 "$SKILL_DIR/scripts/forge-route.py" fallback --plan "$PLAN" --task "$id" --exclude "$3")" || return 1
+  [ -n "$pick" ] || return 1
+  write_escalation "$PLAN" "$id" "$cur" "$pick" fallback || return 1
+  rm -f "$tdir/status" "$tdir/infra.txt"
+  note "$id: $cur is at capacity; falling back to $pick (same tier)"
+}
+
 # The scheduler's `_prepare_retry` target: prepare, then clear the status so the task
 # reads as PENDING again. `retry` clears it itself, after its own preflight. With --resume it
-# prepares an infrastructure stop's resume instead (prepare_resume), and the status stays.
-retry_for_scheduler() { # retry_for_scheduler <plan> <id> [--resume]
+# prepares an infrastructure stop's resume instead (prepare_resume), and the status stays;
+# with --dwarf <spec> the spec is the explicit new implementer; --fallback <pairs> is
+# prepare_fallback.
+retry_for_scheduler() { # retry_for_scheduler <plan> <id> [--resume | --dwarf <spec> | --fallback <pairs>]
   if [ "${3:-}" = --resume ]; then
     prepare_resume "$1" "$2"
     return $?
   fi
-  prepare_retry "$1" "$2" || return $?
+  if [ "${3:-}" = --fallback ]; then
+    prepare_fallback "$1" "$2" "${4:-}"
+    return $?
+  fi
+  # --dwarf: `forge control retry --dwarf`, the same explicit escalation as `retry --dwarf`.
+  if [ "${3:-}" = --dwarf ]; then
+    [ -n "${4:-}" ] || die "_prepare_retry: --dwarf needs a spec"
+    prepare_retry "$1" "$2" "$4" || return $?
+  else
+    prepare_retry "$1" "$2" || return $?
+  fi
   rm -f "$1/tasks/$2/status"
 }
 
@@ -540,7 +568,26 @@ report_guard_notes() { # report_guard_notes <plan>
 
 report_verify_notes() { # report_verify_notes <dir holding verification.*>
   local d="$1"
-  if [ -s "$d/verification.flaky" ]; then
+  if [ -s "$d/verification.lanes.json" ]; then
+    # Lanes: per lane, from the runner's own record
+    python3 - "$d" <<'PY'
+import json, os, sys
+d = sys.argv[1]
+try:
+    with open(os.path.join(d, 'verification.lanes.json'), encoding='utf-8') as fh:
+        lanes = [l for l in json.load(fh).get('lanes') or [] if isinstance(l, dict) and l.get('name')]
+except (OSError, ValueError, AttributeError):
+    sys.exit(0)
+lines = ['FLAKY: lane %s failed once and passed on rerun' % l['name'] for l in lanes if l.get('state') == 'flaky']
+lines += ['known failures only: lane %s' % l['name'] for l in lanes if l.get('state') == 'known']
+failed = [l['name'] for l in lanes if l.get('state') == 'fail']
+if failed:
+    lines.append('verification failed in lane%s %s: read %s' % ('s' if len(failed) > 1 else '', ', '.join(failed),
+                                                                 os.path.join(d, 'verification.log')))
+if lines:
+    print('\n'.join(lines) + '\n')
+PY
+  elif [ -s "$d/verification.flaky" ]; then
     echo "FLAKY verification: the command failed once and passed on rerun — read $d/verification.log"
     echo "  before trusting this result; an intermittent failure can be a real bug."
     echo

package/scripts/forge-lib-verify.sh

@@ -1,12 +1,66 @@
 #!/usr/bin/env bash
 # Verification helpers for forge-parallel.sh. Sourced; functions only.
 
-# The verify command a plan would run: --verify (persisted in the plan) first, then the
-# repository's own .forge/verify. Prints nothing when neither is configured.
+# A lanes directory counts once it holds a lane file (dot files and backups are skipped,
+# as forge-verify.py skips them).
+verify_lanes_present() { # verify_lanes_present <dir>
+  local f
+  [ -d "$1" ] || return 1
+  for f in "$1"/*; do
+    [ -e "$f" ] || continue
+    case "${f##*/}" in *~) continue ;; esac
+    return 0
+  done
+  return 1
+}
+
+# Where a plan's verification comes from, in this order: the plan's lanes (--verify-lane,
+# $PLAN/verify.d/), the plan's --verify, the repository's lanes (.forge/verify.d/), the
+# repository's .forge/verify. Prints "<kind><TAB><location>" with kind lanes:plan,
+# command:plan, lanes:repo or command:repo, or nothing when none is configured.
+verify_source_for() { # verify_source_for <plan> <repo>
+  if verify_lanes_present "$1/verify.d"; then printf 'lanes:plan\t%s\n' "$1/verify.d"
+  elif [ -s "$1/verify_cmd" ]; then printf 'command:plan\t%s\n' "$1/verify_cmd"
+  elif verify_lanes_present "$2/.forge/verify.d"; then printf 'lanes:repo\t%s\n' "$2/.forge/verify.d"
+  elif [ -s "$2/.forge/verify" ]; then printf 'command:repo\t%s\n' "$2/.forge/verify"
+  fi
+  return 0
+}
+
+# The verify command a plan would run, as text: the command itself, or for lanes every
+# lane's command (its #forge: option lines left out), one after another. Prints nothing
+# when nothing is configured.
 verify_command_for() { # verify_command_for <plan> <repo>
-  if [ -s "$1/verify_cmd" ]; then cat "$1/verify_cmd"
-  elif [ -s "$2/.forge/verify" ]; then cat "$2/.forge/verify"
+  local src loc f
+  src="$(verify_source_for "$1" "$2")"
+  loc="${src#*$'\t'}"
+  case "$src" in
+    command:*) cat "$loc" ;;
+    lanes:*)
+      for f in "$loc"/*; do
+        [ -f "$f" ] || continue
+        case "${f##*/}" in *~) continue ;; esac
+        awk 'body || !/^#forge:/ { body = 1; print }' "$f"
+      done ;;
+  esac
+  return 0
+}
+
+# Lanes are checked before any model is paid for: the runner's own validation, against the
+# checkout the lanes will run in (baseline= files exist, reports= and baseline= stay inside).
+# Dies with exit 2 and the runner's messages on an invalid source; a single command, or
+# nothing, passes.
+verify_lanes_check() { # verify_lanes_check <plan> <repo> <checkout>
+  local src loc out
+  src="$(verify_source_for "$1" "$2")"
+  case "$src" in lanes:*) ;; *) return 0 ;; esac
+  loc="${src#*$'\t'}"
+  if [ "${src%%$'\t'*}" = lanes:repo ] && [ -s "$2/.forge/verify" ]; then
+    note ".forge/verify is ignored: the repository declares verification lanes in .forge/verify.d/"
   fi
+  out="$(python3 "$SKILL_DIR/scripts/forge-verify.py" lanes --lanes-dir "$loc" --checkout "$3" 2>&1 >/dev/null)" \
+    || die "invalid verification lanes in $loc:
+$out"
   return 0
 }
 
@@ -15,10 +69,16 @@ verify_command_for() { # verify_command_for <plan> <repo>
 # configuration and say which suites the command does not run. Advisory by design — it
 # prints and records, it never changes an outcome, and a failure of the checker is silent.
 verify_coverage_report() { # verify_coverage_report <checkout> <command> <out-file>
-  local dir="$1" cmd="$2" out="$3" line
+  local dir="$1" cmd="$2" out="$3" line file
   [ -f "$SKILL_DIR/scripts/forge-verify-coverage.py" ] || return 0
   rm -f "$out"
-  if [ -n "$cmd" ]; then
+  if [[ "$cmd" == *$'\n'* ]]; then
+    # Several lines (every lane's command, or a multi-line script): through a file.
+    file="$(mktemp "${TMPDIR:-/tmp}/forge-verify-cmd.XXXXXX")" || return 0
+    printf '%s\n' "$cmd" > "$file"
+    python3 "$SKILL_DIR/scripts/forge-verify-coverage.py" check --repo "$dir" --command-file "$file" --out "$out" >/dev/null 2>&1
+    rm -f "$file"
+  elif [ -n "$cmd" ]; then
     python3 "$SKILL_DIR/scripts/forge-verify-coverage.py" check --repo "$dir" --command="$cmd" --out "$out" >/dev/null 2>&1
   else
     python3 "$SKILL_DIR/scripts/forge-verify-coverage.py" check --repo "$dir" --none --out "$out" >/dev/null 2>&1
@@ -28,3 +88,74 @@ verify_coverage_report() { # verify_coverage_report <checkout> <command> <out-fi
   fi
   return 0
 }
+
+# Lanes mode of verify_result: the lane runner (forge-verify.py) through the same stoppable
+# path as a single command, then verification.* from what it wrote. No whole-command rerun
+# and no Jev triage: the runner reruns a failed lane itself. Returns 0 pass, 1 fail, 130
+# interrupted.
+verify_lanes_run() { # verify_lanes_run <plan> <checkout> <out> <lanes dir> <retries>
+  local plan="$1" wt="$2" out="$3" dir="$4" retries="$5" parallel rc=0 cmd fpcmd
+  parallel="$(cat "$plan/verify_parallel" 2>/dev/null)"
+  case "$parallel" in ''|*[!0-9]*) parallel=3 ;; esac
+  [ "$retries" -le 9 ] 2>/dev/null || retries=9          # the runner takes 0 to 9
+  python3 "$SKILL_DIR/scripts/forge-verify.py" lanes --lanes-dir "$dir" 2>/dev/null \
+    | awk -F'\t' '{ print $1 ": " $3 }' > "$out/verification.command"
+  verify_coverage_report "$wt" "$(verify_command_for "$plan" "$(cat "$plan/repo")")" "$out/verification.coverage.txt"
+  forge_fingerprint "$wt" > "$out/verification.fingerprint" || return 1
+  # The runner judges "a lane changed the checkout" by forge's own fingerprint.
+  fpcmd="source $(printf %q "$SKILL_DIR/scripts/forge-artifact.sh") && forge_fingerprint ."
+  cmd="exec python3 $(printf %q "$SKILL_DIR/scripts/forge-verify.py") run --checkout $(printf %q "$wt")"
+  cmd="$cmd --out $(printf %q "$out") --lanes-dir $(printf %q "$dir") --parallel $parallel --retries $retries"
+  cmd="$cmd --fingerprint-cmd $(printf %q "$fpcmd")"
+  verify_stopped "$out" && return 130
+  # The runner stops its own lanes (TERM, then KILL 5 s later; each lane is a session the
+  # outer KILL cannot reach), so its grace here outlasts that.
+  verify_exec "$wt" "$cmd" "$out/verification.log" 15 || rc=$?
+  [ "$FORGE_VERIFY_STOP" = 1 ] && rc=130
+  echo "$rc" > "$out/verification.exit"
+  # verification.flaky per flaky lane; then each failed lane's last 200 lines under the summary
+  python3 - "$out" <<'PY'
+import json, os, sys
+out = sys.argv[1]
+try:
+    with open(os.path.join(out, 'verification.lanes.json'), encoding='utf-8') as fh:
+        doc = json.load(fh)
+except (OSError, ValueError):
+    sys.exit(0)
+lanes = [l for l in doc.get('lanes') or [] if isinstance(l, dict) and l.get('name')]
+flaky = [l for l in lanes if l.get('state') == 'flaky']
+if flaky:
+    with open(os.path.join(out, 'verification.flaky'), 'w', encoding='utf-8') as fh:
+        fh.write('lanes=%s\n' % ','.join(l['name'] for l in flaky))
+        for l in flaky:
+            runs = l.get('runs') or [{}]
+            fh.write('%s.first_exit=%s\n%s.retries_used=%d\n' % (l['name'], runs[0].get('exit'), l['name'], len(runs) - 1))
+
+def tail(name, lines=200, size=256 << 10):
+    try:
+        with open(os.path.join(out, os.path.basename(name or '')), 'rb') as fh:
+            fh.seek(max(0, os.fstat(fh.fileno()).st_size - size))
+            return fh.read().decode('utf-8', 'replace').splitlines()[-lines:]
+    except OSError:
+        return []
+
+with open(os.path.join(out, 'verification.log'), 'a', encoding='utf-8') as fh:
+    for l in lanes:
+        if l.get('state') != 'fail':
+            continue
+        runs = l.get('runs') or []
+        lines = tail(runs[-1].get('log') if runs else l.get('log'))
+        fh.write('\n== lane %s failed (exit %s); the last %d lines of its output:\n' % (l['name'], l.get('exit'), len(lines)))
+        fh.write(''.join(line + '\n' for line in lines))
+PY
+  case "$rc" in
+    0) echo PASS > "$out/verification.status" ;;
+    130) verify_mark_interrupted "$out"; return 130 ;;
+    *) echo FAIL > "$out/verification.status" ;;
+  esac
+  local flaky; flaky="$(sed -n 's/^lanes=//p' "$out/verification.flaky" 2>/dev/null)"
+  [ -z "$flaky" ] || note "FLAKY: lane ${flaky//,/, lane } failed once and passed on rerun; this is not a clean pass"
+  [ "$rc" = 0 ] && return 0
+  note "verification failed ($( [ "$rc" = 2 ] && echo 'the lane runner refused its configuration' || echo 'see the lanes below' )): $out/verification.log"
+  return 1
+}

package/scripts/forge-parallel-options.sh

@@ -21,15 +21,46 @@
 #   --timeout S             run retry         $PLAN/timeout        dispatch timeout, seconds (0 = none)
 #   --retry-failed N        run               (this run only)      automatic reviewer-driven retries
 #   --infra-retries M       run               (this run only)      waits out quota/auth/network stops
+#   --infra-backoff SHAPE   run               $PLAN/infra_backoff  waits between those stops: exp:B[,C],
+#                                                                  linear:S,T[,C] or list:a,b,... (default exp:60)
+#   --infra-fallback        run               $PLAN/infra_fallback a dwarf's capacity stop moves it to the next healthy
+#   --no-infra-fallback                                            model of its tier pool (off by default)
 #   --qa-threshold Pn|none  run retry         $PLAN/qa_threshold   least severe finding that still blocks
 #   --verify-retries N      run retry integrate  $PLAN/verify_retries  reruns of a failed verify command
 #   --escalate-after N      run retry         $PLAN/escalate_after failed attempts on one model before the
 #                                                                  next goes a tier up (default 3; 0 = off)
+#   --escalate-effort       run retry         $PLAN/escalate_effort  on: before the stop, retry the same
+#                                                                  model at the next effort its pairing accepts;
+#                                                                  tasks already stopped are decided again
+#   --no-escalate-effort    run retry         (removes the file)   turn that step off (the default)
 #   --final-review SPEC     integrate         $PLAN/final_qa       QA spec for the whole-run review
+#   --verify-lane L=CMD     run retry         $PLAN/verify.d/<L>   a verification lane: name[options]=command,
+#                                                                  options as in a lane file (group=g,first,...);
+#                                                                  repeatable; replaces --verify and the lanes
+#                                                                  of an earlier invocation
+#   --verify-parallel N     run retry integrate  $PLAN/verify_parallel  lanes run at once (1-32, default 3)
 PP_SHIFT=2
 PP_RETRY_FAILED=0
 PP_INFRA_RETRIES=0
 PP_FINAL_REVIEW=""
+PP_LANES_FRESH=0   # the first --verify-lane of an invocation, or the first after a --verify, starts a fresh verify.d/
+
+# --verify-lane 'name[opt,opt]=command' -> $PLAN/verify.d/name: the options become the lane
+# file's first line, `#forge: opt opt`, and the command follows, as in a lane file.
+pp_lane() { # pp_lane <plan> <argument>
+  local plan="$1" arg="$2" re='^([A-Za-z0-9][A-Za-z0-9_-]{0,63})(\[([^]]*)\])?=(.*)$'
+  [[ "$arg" =~ $re ]] \
+    || die "--verify-lane '$arg': expected name[options]=command (a name of letters, digits, _ and -, at most 64; options in closed brackets)"
+  local name="${BASH_REMATCH[1]}" opts="${BASH_REMATCH[3]}" cmd="${BASH_REMATCH[4]}"
+  [ -n "${cmd//[[:space:]]/}" ] || die "--verify-lane '$arg': the command is empty"
+  if [ "$PP_LANES_FRESH" = 0 ]; then
+    rm -rf "$plan/verify.d" && mkdir "$plan/verify.d" || die "could not create $plan/verify.d"
+    PP_LANES_FRESH=1
+  fi
+  { [ -z "$opts" ] || printf '#forge: %s\n' "${opts//,/ }"; printf '%s\n' "$cmd"; } > "$plan/verify.d/$name" \
+    || die "could not write $plan/verify.d/$name"
+  rm -f "$plan/verify_cmd"
+}
 
 pp_nonneg() { # pp_nonneg <flag> <value>
   case "$2" in ''|*[!0-9]*) die "$1 needs a non-negative integer" ;; esac
@@ -49,6 +80,16 @@ forge_pp_flag() {
     --infra-retries)
       [ "$cmd" = run ] || return 1
       pp_nonneg --infra-retries "${2:-}"; PP_INFRA_RETRIES="$2" ;;
+    --infra-fallback)
+      [ "$cmd" = run ] || return 1
+      PP_SHIFT=1; printf 'on\n' > "$plan/infra_fallback" ;;
+    --no-infra-fallback)
+      [ "$cmd" = run ] || return 1
+      PP_SHIFT=1; rm -f "$plan/infra_fallback" ;;
+    --infra-backoff)
+      [ "$cmd" = run ] || return 1
+      PP_MSG="$(python3 "$SKILL_DIR/scripts/forge-schedule.py" backoff-check "${2:-}" 2>&1)" || die "${PP_MSG#forge: }"
+      printf '%s\n' "$2" > "$plan/infra_backoff" ;;
     --qa-threshold)
       case "$cmd" in run|retry|review) ;; *) return 1 ;; esac
       case "${2:-}" in
@@ -64,10 +105,28 @@ forge_pp_flag() {
       if [ "$2" = 0 ]; then
         rm -f "$plan"/tasks/*/escalation.stop "$plan"/tasks/*/split.suggested "$plan"/tasks/*/escalation.next
       fi ;;
+    --escalate-effort)
+      case "$cmd" in run|retry) ;; *) return 1 ;; esac
+      PP_SHIFT=1                                  # a switch, not --escalate-after's N
+      printf 'on\n' > "$plan/escalate_effort"
+      # A task the policy stopped with the step off is decided again: the step may have a move.
+      rm -f "$plan"/tasks/*/escalation.stop "$plan"/tasks/*/split.suggested "$plan"/tasks/*/escalation.next ;;
+    --no-escalate-effort)
+      case "$cmd" in run|retry) ;; *) return 1 ;; esac
+      PP_SHIFT=1
+      rm -f "$plan/escalate_effort" ;;
     --verify-retries)
       case "$cmd" in run|retry|integrate) ;; *) return 1 ;; esac
       pp_nonneg --verify-retries "${2:-}"
       printf '%s' "$2" > "$plan/verify_retries" ;;
+    --verify-lane)
+      case "$cmd" in run|retry) ;; *) return 1 ;; esac
+      pp_lane "$plan" "${2-}" ;;
+    --verify-parallel)
+      case "$cmd" in run|retry|integrate) ;; *) return 1 ;; esac
+      case "${2:-}" in ''|*[!0-9]*) die "--verify-parallel needs an integer from 1 to 32" ;; esac
+      [ "$2" -ge 1 ] && [ "$2" -le 32 ] || die "--verify-parallel needs an integer from 1 to 32"
+      printf '%s' "$2" > "$plan/verify_parallel" ;;
     --final-review)
       [ "$cmd" = integrate ] || return 1
       PP_FINAL_REVIEW="${2:?--final-review needs a QA spec}"