@giamme/forge 0.8.0 → 0.10.0

package/scripts/forge-runs.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""forge runs [--repo DIR | --all] [--limit N] [--json]: this repo's recent runs, in the terminal.
+
+Every solo run, `plan` and `run` registers its run dir (forge_registry.py); this lists them newest
+first, one line each (run id, kind, age, task states, verification and final review, goal) with the
+run dir under it, so you can see what happened without a browser and get the dir to resume, retry
+or open with `forge ui open`. The repo is the Git top level of the current directory unless
+`--repo` names one; `--all` lists the runs of every repo. It reads the same run summaries as
+forge-ui and never writes anything.
+
+`forge runs rm <run-dir> [--dry-run] [--yes] [--no-archive] [--worktrees] [--branches] [--json]`
+is the one thing here that changes anything: it deletes a run, after its traces are in the dataset
+and an archive exists (scripts/forge-runs-rm.py).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+
+SKILL = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(SKILL / 'scripts'))
+sys.path.insert(0, str(SKILL / 'ui'))
+
+import forge_registry  # noqa: E402
+from forge_ui import reader  # noqa: E402
+
+FIELDS = ('key', 'run_id', 'kind', 'goal', 'repo', 'path', 'created', 'updated', 'task_count', 'counts',
+          'verification_status', 'final_review')
+
+
+def repo_of(start: Path) -> Path:
+    """The Git top level of `start`, else `start` itself: the key runs are registered under."""
+    try:
+        out = subprocess.run(['git', '-C', str(start), 'rev-parse', '--show-toplevel'], capture_output=True,
+                             text=True, stdin=subprocess.DEVNULL, timeout=10)
+        if out.returncode == 0 and out.stdout.strip():
+            return Path(out.stdout.strip()).resolve()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return start.resolve()
+
+
+def registered(path: Path) -> list[str]:
+    """The run dirs one registry file lists (absolute paths, comments skipped)."""
+    try:
+        lines = path.read_text(encoding='utf-8', errors='replace').splitlines()
+    except OSError:
+        return []
+    return [line.strip() for line in lines if line.strip() and not line.startswith('#') and os.path.isabs(line.strip())]
+
+
+def runs(repo: Path | None) -> list[dict]:
+    if repo is None:
+        found = reader.discover(registry=True, registry_dir=forge_registry.registry_dir())
+    else:
+        found = reader.discover(plans=registered(forge_registry.runs_registry(repo)), registry=False)
+    result = []
+    for path, source in found:
+        try:
+            summary = reader.summarize(path, source)
+        except Exception:  # one unreadable run dir must not hide the others
+            continue
+        result.append({k: summary.get(k) for k in FIELDS})
+    return sorted(result, key=lambda r: r['updated'] if r['updated'] is not None else float('-inf'), reverse=True)
+
+
+def age(epoch, now) -> str:
+    if epoch is None:
+        return '?'
+    s = max(0, now - epoch)
+    for unit, size in (('d', 86400), ('h', 3600), ('m', 60)):
+        if s >= size:
+            return f'{int(s // size)}{unit} ago'
+    return 'just now'
+
+
+def line(run: dict, now: float, with_repo: bool = False) -> str:
+    counts = ' · '.join(f'{status.lower()} {n}' for status, n in (run['counts'] or {}).items()) or 'no tasks'
+    checks = ''.join(f'  {label} {run[key]}' for label, key in (('verify', 'verification_status'), ('review', 'final_review'))
+                     if run.get(key))
+    goal = str(run['goal'] or 'Untitled run').splitlines()[0]
+    if len(goal) > 72:
+        goal = goal[:71] + '…'
+    repo = f"  {Path(run['repo']).name or run['repo']}" if with_repo and run.get('repo') else ''
+    return f"{run['run_id'] or run['key']}  {run['kind']}{repo}  {age(run['updated'], now)}  {counts}{checks}  {goal}\n    {run['path']}"
+
+
+def remove(argv) -> int:
+    """`forge runs rm`: loaded on demand, so listing never pays for the dataset package."""
+    import importlib.util
+    spec = importlib.util.spec_from_file_location('forge_runs_rm', Path(__file__).resolve().with_name('forge-runs-rm.py'))
+    module = importlib.util.module_from_spec(spec)
+    sys.modules['forge_runs_rm'] = module
+    spec.loader.exec_module(module)
+    return module.main(argv)
+
+
+def main(argv=None) -> int:
+    argv = sys.argv[1:] if argv is None else list(argv)
+    if argv[:1] == ['rm']:
+        return remove(argv[1:])
+    parser = argparse.ArgumentParser(prog='forge runs', description="This repo's recent forge runs, newest first. Listing is read-only.",
+                                     epilog='Delete a run, keeping its traces: forge runs rm <run-dir> (forge runs rm --help)')
+    which = parser.add_mutually_exclusive_group()
+    which.add_argument('--repo', type=Path, metavar='DIR', help='the repo whose runs to list (default: the Git top level of the current directory)')
+    which.add_argument('--all', action='store_true', help='list the runs of every repo')
+    parser.add_argument('--limit', type=int, default=10, metavar='N', help='at most N runs (default 10; 0 lists them all)')
+    parser.add_argument('--json', action='store_true', help='print the runs as a JSON array, for scripts and agents')
+    args = parser.parse_args(argv)
+    if args.limit < 0:
+        parser.error('--limit must be 0 or more')
+    if args.repo is not None and not args.repo.is_dir():
+        parser.error(f'--repo: no such directory: {args.repo}')
+    repo = None if args.all else repo_of(args.repo if args.repo is not None else Path.cwd())
+    found = runs(repo)
+    shown = found[:args.limit] if args.limit else found
+    if args.json:
+        json.dump(shown, sys.stdout, indent=2)
+        sys.stdout.write('\n')
+        return 0
+    where = 'every repo' if repo is None else str(repo)
+    if not found:
+        print(f'forge runs: no runs registered for {where}')
+        print('Runs register when they start (/forge "<goal>" --dwarf <model>); FORGE_RUN_REGISTRY=off turns that off.')
+        return 0
+    print(f'forge runs for {where}: {len(shown)} of {len(found)}, newest first')
+    now = time.time()
+    for run in shown:
+        print(line(run, now, with_repo=repo is None))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/forge-runtime.py

@@ -101,9 +101,45 @@ def token_count(value: object) -> int | None:
     return value if type(value) is int and value >= 0 else None
 
 
+# Harnesses whose log is a stream of JSON events rather than the answer itself.
+STRUCTURED = ('codex', 'claude', 'openclaude', 'grok', 'opencode')
+
+
+def _opencode_final(events: list) -> str:
+    """opencode's answer: the text of its last step, after its last tool call."""
+    final, step = '', []
+    for event in events:
+        kind = event.get('type')
+        if kind in ('step_start', 'tool_use'):
+            step = []
+        elif kind == 'text':
+            part = event.get('part')
+            text = part.get('text') if isinstance(part, dict) else None
+            if isinstance(text, str) and text.strip():
+                step.append(text)
+                final = '\n'.join(step)
+    return final
+
+
 def extract(log: str, harness: str) -> tuple[str | None, dict[str, int | None]]:
     usage = dict(input_tokens=None, cached_input_tokens=None, output_tokens=None)
     final = None
+    if harness == 'opencode':
+        events = _events(log)[0]
+        for event in events:
+            part = event.get('part')
+            tokens = part.get('tokens') if event.get('type') == 'step_finish' and isinstance(part, dict) else None
+            if not isinstance(tokens, dict):
+                continue
+            cache = tokens.get('cache') if isinstance(tokens.get('cache'), dict) else {}
+            # input counts what was read from and written to the cache, as claude's does
+            for key, parts in (('input_tokens', (tokens.get('input'), cache.get('read'), cache.get('write'))),
+                               ('cached_input_tokens', (cache.get('read'),)),
+                               ('output_tokens', (tokens.get('output'), tokens.get('reasoning')))):
+                counts = [token_count(v) for v in parts if v is not None]
+                if counts and all(v is not None for v in counts):
+                    usage[key] = (usage[key] or 0) + sum(counts)
+        return _opencode_final(events) or None, usage
     for line in log.splitlines():
         try:
             event = json.loads(line)
@@ -124,8 +160,9 @@ def extract(log: str, harness: str) -> tuple[str | None, dict[str, int | None]]:
             if isinstance(item, dict) and item.get('type') == 'agent_message':
                 if isinstance(item.get('text'), str):
                     final = item['text']
-        # grok's streaming-messages-json ends in the same `result` event, token buckets included.
-        if harness in ('claude', 'grok') and event.get('type') == 'result':
+        # claude's and openclaude's stream-json and grok's streaming-messages-json end in the same
+        # `result` event, token buckets included.
+        if harness in ('claude', 'openclaude', 'grok') and event.get('type') == 'result':
             if isinstance(event.get('result'), str):
                 final = event['result']
             # Claude reports uncached, cache-read and cache-creation separately.
@@ -149,7 +186,7 @@ def finish(attempt: str, root: str, role: str, harness: str, rc: str) -> None:
 def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None:
     log = root / (role + '.log')
     last = root / (role + '.last')
-    structured = harness in ('codex', 'claude', 'grok')
+    structured = harness in STRUCTURED
     text = log.read_text(errors='replace') if structured and log.exists() else ''
     final, usage = extract(text, harness)
     if not last.exists() or not last.stat().st_size:
@@ -439,7 +476,7 @@ def classify(log: str, harness: str, rc: object, last: str = '', role: str = '',
         text = Path(log).read_text(errors='replace')
     except OSError:
         text = ''
-    events, stray = _events(text) if harness in ('codex', 'claude', 'openclaude', 'grok') else ([], [])
+    events, stray = _events(text) if harness in STRUCTURED else ([], [])
 
     if harness == 'codex':
         final = _codex_final(events, last)
@@ -454,6 +491,22 @@ def classify(log: str, harness: str, rc: object, last: str = '', role: str = '',
             return 'empty', None, 'codex finished with exit 0 but produced no final message'
         return classify_text('\n'.join(pending + [_tail(stray)]), now=now)
 
+    if harness == 'opencode' and events:
+        # Like codex: an error event counts when the run produced no answer after it.
+        errors = []
+        for event in events:
+            error = event.get('error') if event.get('type') == 'error' else None
+            if isinstance(error, dict):
+                data = error.get('data') if isinstance(error.get('data'), dict) else {}
+                errors.append(str(data.get('message') or error.get('name') or json.dumps(error)))
+        final = _opencode_final(events)
+        if rc == 0:
+            if final.strip():
+                return None
+            hit = classify_text('\n'.join(errors), now=now) if errors else None
+            return hit or ('empty', None, 'opencode finished with exit 0 but produced no final message')
+        return classify_text('\n'.join(errors + [_tail(stray)]), now=now)
+
     results = [e for e in events if e.get('type') == 'result'] if harness in ('claude', 'openclaude', 'grok') else []
     if results:
         event = results[-1]
@@ -478,8 +531,8 @@ def classify(log: str, harness: str, rc: object, last: str = '', role: str = '',
             return 'empty', None, '%s finished with exit 0 but its final result is empty' % harness
         return None
 
-    # Raw-log harness (opencode, antigravity, openclaude's plain text, or a claude that
-    # died before emitting its result): the log is the message, so only the tail of a
+    # Raw-log harness (antigravity, an opencode or openclaude from before their streams, or a
+    # claude that died before emitting its result): the log is the message, so only the tail of a
     # failing run is evidence. A successful run's prose is never scanned.
     if rc != 0:
         return classify_text(_tail(text), now=now)

package/scripts/forge-schedule.py

@@ -10,17 +10,26 @@ Two opt-in behaviours live here, both off by default so a plain `run` is unchang
   network, empty output) up to M times instead of stopping. Without it the scheduler
   stops dispatching, lets running tasks drain and exits 8: nothing was the task's fault
   and no attempt was spent, so the next `run` simply resumes.
+
+Both numbers are per invocation, but while the scheduler runs they are mirrored into the
+plan dir (retry_budget, infra_budget, tasks/<id>/retry_budget) and re-read at every
+decision, so `forge control` can change them mid-run. See "live control files" below.
 """
 from __future__ import annotations
 
 import argparse
+import contextlib
+import errno
 import fcntl
+import json
 import os
 from pathlib import Path
 import posixpath
 import signal
+import stat
 import subprocess
 import sys
+import tempfile
 import time
 
 
@@ -80,6 +89,157 @@ def infra_wait(info: dict, cycle: int) -> float:
     return min(base * (2 ** cycle), max(base, 1800.0))
 
 
+# --- live control files ------------------------------------------------------------------
+# Created at start and removed at exit, so a later `run` never inherits a budget it did not
+# ask for. Writers replace files atomically, so readers need no lock; the control lock only
+# orders a lifecycle write against `forge control`'s liveness check. It is never held across
+# a model dispatch and never taken together with schedule.lock.
+CONTROL_LOCK_WAIT = 5.0   # seconds; a busy lock must never fail a run
+SWEEP_EVERY = 1.0         # seconds between looks at tasks that already ended FAIL/UNKNOWN
+
+
+def _warn(message: str) -> None:
+    print('forge: ' + message, file=sys.stderr, flush=True)
+
+
+def _task_id_ok(name: str) -> bool:
+    return bool(name) and name not in ('.', '..') and '/' not in name and '\0' not in name
+
+
+def _resolve(plan: object, *parts: str) -> str:
+    """plan/<parts> if every directory on the way is a real one (no symlink) inside the
+    canonical plan dir and an existing target is a regular file. OSError otherwise."""
+    if not all(_task_id_ok(part) for part in parts):
+        raise OSError(errno.EINVAL, 'invalid path component', '/'.join(parts))
+    path = os.path.realpath(plan)
+    for part in parts[:-1]:
+        path = os.path.join(path, part)
+        if not stat.S_ISDIR(os.lstat(path).st_mode):
+            raise OSError(errno.ENOTDIR, 'not a real directory', path)
+    path = os.path.join(path, parts[-1])
+    try:
+        if not stat.S_ISREG(os.lstat(path).st_mode):
+            raise OSError(errno.EPERM, 'not a regular file', path)
+    except FileNotFoundError:
+        pass
+    return path
+
+
+def _write_atomic(plan: object, text: str, *parts: str) -> bool:
+    """Temp file in the same directory, then rename. A failure warns and returns False."""
+    tmp = None
+    try:
+        path = _resolve(plan, *parts)
+        fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix='.' + parts[-1] + '.')
+        with os.fdopen(fd, 'w') as handle:
+            os.fchmod(handle.fileno(), 0o644)
+            handle.write(text)
+        os.replace(tmp, path)
+        return True
+    except OSError as error:
+        _warn('could not write ' + '/'.join(parts) + ': ' + str(error))
+        if tmp:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp)
+        return False
+
+
+def _remove(plan: object, *parts: str) -> None:
+    try:
+        os.unlink(_resolve(plan, *parts))
+    except FileNotFoundError:
+        pass
+    except OSError as error:
+        _warn('could not remove ' + '/'.join(parts) + ': ' + str(error))
+
+
+def _read_budget(plan: object, *parts: str) -> int | None:
+    """A non-negative integer from a control file; None when it is missing, empty, a
+    symlink or anything else, so the caller falls through to its next source."""
+    try:
+        fd = os.open(_resolve(plan, *parts), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+        try:
+            data = os.read(fd, 65)
+        finally:
+            os.close(fd)
+        text = data.decode('ascii').strip()
+    except (OSError, UnicodeDecodeError):
+        return None
+    return int(text) if len(data) <= 64 and text.isdigit() else None
+
+
+@contextlib.contextmanager
+def _control_lock(plan: object):
+    """flock on control.lock, retried for CONTROL_LOCK_WAIT seconds. On timeout or any
+    error: warn and proceed without it (yielding False), never fail the run."""
+    fd = None
+    try:
+        fd = os.open(_resolve(plan, 'control.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o644)
+        deadline = time.monotonic() + CONTROL_LOCK_WAIT
+        while True:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() >= deadline:
+                    raise OSError(errno.EWOULDBLOCK, 'busy for ' + ('%g' % CONTROL_LOCK_WAIT) + 's')
+                time.sleep(0.05)
+    except OSError as error:
+        _warn('control.lock not taken, continuing without it: ' + str(error))
+        if fd is not None:
+            os.close(fd)
+            fd = None
+    try:
+        yield fd is not None
+    finally:
+        if fd is not None:
+            os.close(fd)
+
+
+def _task_dirs(plan: object, names: object) -> list[str]:
+    """The planned tasks plus any other task dir, since `forge control` accepts an id
+    that is not in tasks.tsv and its file must not outlive the invocation either."""
+    found = set(names)
+    try:
+        with os.scandir(os.path.join(os.path.realpath(plan), 'tasks')) as entries:
+            found.update(e.name for e in entries if _task_id_ok(e.name) and e.is_dir(follow_symlinks=False))
+    except OSError:
+        pass
+    return sorted(found)
+
+
+def _pinned(plan: object, name: str) -> bool:
+    try:
+        return os.path.lexists(_resolve(plan, 'tasks', name, 'escalation.pin'))
+    except OSError:
+        return False
+
+
+def live_start(plan: object, names: object, retry_failed: int, infra_retries: int) -> None:
+    """Publish the flags and that this scheduler is live. Stale per-task overrides and
+    usage counters from an earlier invocation go first."""
+    with _control_lock(plan):
+        for name in _task_dirs(plan, names):
+            _remove(plan, 'tasks', name, 'retry_budget')
+            _remove(plan, 'tasks', name, 'retries.used')
+        _write_atomic(plan, str(retry_failed), 'retry_budget')
+        _write_atomic(plan, str(infra_retries), 'infra_budget')
+        _write_atomic(plan, json.dumps({'pid': os.getpid(), 'started': time.time()}), 'scheduler.live')
+
+
+def live_stop(plan: object, names: object) -> None:
+    """Withdraw liveness first, then every budget. retries.used stays for the UI."""
+    try:
+        with _control_lock(plan):
+            _remove(plan, 'scheduler.live')
+            _remove(plan, 'retry_budget')
+            _remove(plan, 'infra_budget')
+            for name in _task_dirs(plan, names):
+                _remove(plan, 'tasks', name, 'retry_budget')
+    except Exception as error:
+        _warn('could not withdraw the live control files: ' + str(error))
+
+
 def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
              infra_retries: int = 0) -> int:
     if capacity <= 0:
@@ -97,16 +257,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
     if len(tasks) != len(rows):
         raise ValueError('duplicate task IDs')
     for name in tasks:
-        if not name or name in ('.', '..') or '/' in name:
+        if not _task_id_ok(name):
             raise ValueError('invalid task ID: ' + name)
         (plan / 'tasks' / name).mkdir(parents=True, exist_ok=True)
     active = {}
     pending = list(tasks)
     interrupted = False
     retries = {}       # automatic reviewer-driven retries used per task, this run
+    refused = {}       # task -> budget at the time its retry could not be prepared
     retried = set()    # tasks whose next dispatch is a retry (FORGE_FRACTAL_RETRY=1)
     offsets = {}       # how much of each task.out was already echoed under --output full
     infra_seen = []    # tasks stopped by an infrastructure failure, awaiting resume
+    held = set()       # INFRA tasks not resumed: their pin could not be applied
     infra_cycles = 0   # how many times this run already waited out an infrastructure pause
     paused = False     # an infrastructure failure was seen: dispatch nothing new
     resume_at = 0.0
@@ -127,30 +289,61 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
         if result.returncode and status(name) == 'PASS':
             mark(name, 'ERROR')
 
+    def budget(name: str) -> int:
+        """Automatic retries this task may use. Read at every decision, never cached: the
+        task's own file, then the run-wide file, then the flag."""
+        for parts in (('tasks', name, 'retry_budget'), ('retry_budget',)):
+            value = _read_budget(plan, *parts)
+            if value is not None:
+                return value
+        return retry_failed
+
+    def infra_budget() -> int:
+        value = _read_budget(plan, 'infra_budget')
+        return infra_retries if value is None else value
+
     def retryable(name: str) -> bool:
+        root = plan / 'tasks' / name
         return (
-            retry_failed > 0
-            and retries.get(name, 0) < retry_failed
-            and status(name) in ('FAIL', 'UNKNOWN')
+            status(name) in ('FAIL', 'UNKNOWN')
+            and retries.get(name, 0) < budget(name)
             # A task that produced no new work (empty diff, byte-identical retry) would
             # only repeat itself; its marker says so and it is left for a human.
-            and not (plan / 'tasks' / name / 'noretry').exists()
+            and not (root / 'noretry').exists()
             # The retry policy found no stronger model after repeated failures on one: a
-            # split, or an explicit `retry --dwarf`, is the human's call.
-            and not (plan / 'tasks' / name / 'escalation.stop').exists()
+            # split, or an explicit `retry --dwarf`, is the human's call. A pin is exactly
+            # that call, made ahead of time: the next full retry applies it.
+            and (not (root / 'escalation.stop').exists() or _pinned(plan, name))
         )
 
     def requeue(name: str) -> bool:
         """Prepare a retry exactly as `retry` does, then put the task at the queue head."""
         result = subprocess.run(['/bin/bash', script, '_prepare_retry', str(plan), name])
         if result.returncode:
+            refused[name] = budget(name)
             return False
+        refused.pop(name, None)
         retries[name] = retries.get(name, 0) + 1
+        _write_atomic(plan, str(retries[name]), 'tasks', name, 'retries.used')
         retried.add(name)
         if name not in pending:
             pending.insert(0, name)
         print('forge: retrying ' + name + ' (automatic retry ' + str(retries[name]) + '/'
-              + str(retry_failed) + ')', file=sys.stderr, flush=True)
+              + str(budget(name)) + ')', file=sys.stderr, flush=True)
+        return True
+
+    def resume(name: str) -> bool:
+        """Prepare an INFRA task's next entry. Not a retry, so nothing is counted, but a
+        pending pin applies when that entry is a full implementer attempt. False when a
+        claimed pin could not be put in place: the task then stays INFRA, as `retry` would."""
+        if not _pinned(plan, name):
+            return True
+        result = subprocess.run(['/bin/bash', script, '_prepare_retry', str(plan), name, '--resume'])
+        if result.returncode:
+            held.add(name)
+            print('forge: ' + name + ' is not resumed: its escalation pin could not be applied',
+                  file=sys.stderr, flush=True)
+            return False
         return True
 
     def echo_output(name: str) -> None:
@@ -171,6 +364,7 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
     signal.signal(signal.SIGTERM, stop)
     signal.signal(signal.SIGINT, stop)
     try:
+        live_start(plan, tasks, retry_failed, infra_retries)
         # Failures require an explicit retry unless this run asked for automatic ones.
         # Saved PASS results use the same fingerprint checks without paying for
         # implementation or QA again; an INFRA stop is not a failure of the work, so it
@@ -183,13 +377,19 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                 mark(name, 'INTERRUPTED')
             if state in ('FAIL', 'UNKNOWN') and retryable(name):
                 requeue(name)
+            if state == 'INFRA' and not resume(name):
+                pending.remove(name)
+                continue
             if status(name) not in ('PENDING', 'BLOCKED', 'INFRA'):
                 pending.remove(name)
+        swept = time.monotonic()
         while pending or active:
             if interrupted:
                 break
+            ended = set()
             for name, (process, log) in list(active.items()):
                 if process.poll() is not None:
+                    ended.add(name)
                     log.close()
                     del active[name]
                     if os.environ.get('FORGE_OUTPUT') == 'full':
@@ -205,7 +405,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                             infra_seen.append(name)
                     elif state in ('FAIL', 'UNKNOWN') and retryable(name):
                         requeue(name)
-            if paused and not active and infra_seen and infra_cycles < infra_retries:
+            if time.monotonic() - swept >= SWEEP_EVERY:
+                # A human can raise a budget after a task already ended FAIL while others
+                # still run. The initial pass's predicate, for tasks nothing else will look
+                # at again; one whose retry could not be prepared waits for a new budget.
+                swept = time.monotonic()
+                for name in tasks:
+                    if name in pending or name in active or name in ended:
+                        continue
+                    if status(name) in ('FAIL', 'UNKNOWN') and retryable(name) \
+                            and refused.get(name) != budget(name):
+                        requeue(name)
+            if paused and not active and infra_seen and infra_cycles < infra_budget():
                 # Everything running has drained. Wait out the outage once for all the
                 # tasks it stopped, then dispatch them again.
                 wait = max(infra_wait(infra_info(plan / 'tasks' / n), infra_cycles) for n in infra_seen)
@@ -218,8 +429,10 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                     time.sleep(min(0.05, max(resume_at - time.time(), 0)))
                 if interrupted:
                     break
+                # A pin set before or during the wait applies to the entries that run the
+                # implementer again; a QA-stage stop keeps it for the next full retry.
                 for name in reversed(infra_seen):
-                    if name not in pending:
+                    if name not in pending and resume(name):
                         pending.insert(0, name)
                 infra_seen.clear()
                 paused = False
@@ -249,15 +462,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                 pending.remove(name)
                 print('forge: dispatched ' + name + ' -> ' + str(root), file=sys.stderr, flush=True)
             if not active:
-                if paused:
+                if paused or held:
                     # Infrastructure stop: leave everything not yet run PENDING. Marking it
                     # BLOCKED would turn "the provider was down" into a recorded failure.
+                    # A task held back by its pin is still an infrastructure stop.
                     break
                 for name in pending:
                     mark(name, 'BLOCKED')
                 break
             time.sleep(0.05)
     finally:
+        # Not live from here on: a late `forge control` must not edit budgets nobody reads.
+        live_stop(plan, tasks)
         for process, log in active.values():
             try:
                 os.killpg(process.pid, signal.SIGTERM)