@giamme/forge 0.0.0-stage → 0.9.0

package/scripts/forge-schedule.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""Readiness scheduler. Only the coordinator merges; tasks keep pinned baselines.
+
+Two opt-in behaviours live here, both off by default so a plain `run` is unchanged:
+
+* ``--retry-failed N`` re-queues a task whose reviewer said FAIL/UNKNOWN up to N more
+  times *inside the same run*, so one failing task never holds the others back until the
+  run exits. Dependents already wait for MERGED, so nothing downstream starts early.
+* ``--infra-retries M`` waits out an infrastructure failure (quota, auth, rate limit,
+  network, empty output) up to M times instead of stopping. Without it the scheduler
+  stops dispatching, lets running tasks drain and exits 8: nothing was the task's fault
+  and no attempt was spent, so the next `run` simply resumes.
+"""
+from __future__ import annotations
+
+import argparse
+import fcntl
+import os
+from pathlib import Path
+import posixpath
+import signal
+import subprocess
+import sys
+import time
+
+
+def dataset_event(name: str, plan: Path, task: str, values: dict) -> None:
+    """Record a status only the scheduler sets (INTERRUPTED, BLOCKED, ERROR) in the run
+    dataset. No task process is alive to record it. Fail-open, like every capture."""
+    try:
+        scripts = str(Path(__file__).resolve().parent)
+        if scripts not in sys.path:
+            sys.path.insert(0, scripts)
+        from forge_dataset.capture import point
+        point(name, str(plan), task, values)
+    except Exception:
+        pass
+
+
+def overlap(a: str, b: str) -> bool:
+    def paths(value: str) -> list[str]:
+        return [posixpath.normpath(p) for p in value.split(',') if p and p != '-']
+
+    return any(
+        x == '.' or y == '.' or x == y or x.startswith(y + '/') or y.startswith(x + '/')
+        for x in paths(a) for y in paths(b)
+    )
+
+
+INFRA_EXIT = 8
+# A parsed retry-after beyond this is not a pause worth waiting for inside one run.
+INFRA_WAIT_CAP = 6 * 3600
+
+
+def infra_info(root: Path) -> dict:
+    """key=value lines the task wrote when it stopped on an infrastructure failure."""
+    info = {}
+    try:
+        for line in (root / 'infra.txt').read_text(errors='replace').splitlines():
+            key, _, value = line.partition('=')
+            if key:
+                info[key.strip()] = value.strip()
+    except OSError:
+        pass
+    return info
+
+
+def infra_wait(info: dict, cycle: int) -> float:
+    """Seconds to wait before re-dispatching after an infrastructure failure."""
+    try:
+        base = float(os.environ.get('FORGE_INFRA_BACKOFF', '60'))
+    except ValueError:
+        base = 60.0
+    try:
+        hinted = float(info.get('retry_after', ''))
+    except ValueError:
+        hinted = None
+    if hinted is not None and hinted >= 0:
+        return min(hinted, INFRA_WAIT_CAP)
+    return min(base * (2 ** cycle), max(base, 1800.0))
+
+
+def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
+             infra_retries: int = 0) -> int:
+    if capacity <= 0:
+        raise ValueError('capacity must be positive')
+    if retry_failed < 0 or infra_retries < 0:
+        raise ValueError('retry counts must not be negative')
+    plan = Path(plan)
+    rows = [
+        line.split('\t') for line in (plan / 'tasks.tsv').read_text().splitlines()
+        if line and not line.startswith('#')
+    ]
+    if any(len(row) != 7 for row in rows):
+        raise ValueError('tasks.tsv requires seven columns')
+    tasks = {row[0]: row for row in rows}
+    if len(tasks) != len(rows):
+        raise ValueError('duplicate task IDs')
+    for name in tasks:
+        if not name or name in ('.', '..') or '/' in name:
+            raise ValueError('invalid task ID: ' + name)
+        (plan / 'tasks' / name).mkdir(parents=True, exist_ok=True)
+    active = {}
+    pending = list(tasks)
+    interrupted = False
+    retries = {}       # automatic reviewer-driven retries used per task, this run
+    retried = set()    # tasks whose next dispatch is a retry (FORGE_FRACTAL_RETRY=1)
+    offsets = {}       # how much of each task.out was already echoed under --output full
+    infra_seen = []    # tasks stopped by an infrastructure failure, awaiting resume
+    infra_cycles = 0   # how many times this run already waited out an infrastructure pause
+    paused = False     # an infrastructure failure was seen: dispatch nothing new
+    resume_at = 0.0
+
+    def status(name: str) -> str:
+        root = plan / 'tasks' / name
+        if (root / 'merged').exists():
+            return 'MERGED'
+        file = root / 'status'
+        return file.read_text().strip() if file.exists() else 'PENDING'
+
+    def mark(name: str, state: str) -> None:
+        (plan / 'tasks' / name / 'status').write_text(state + '\n')
+        dataset_event('sched_terminal', plan, name, dict(status=state))
+
+    def merge(name: str) -> None:
+        result = subprocess.run(['/bin/bash', script, '_merge', str(plan), name])
+        if result.returncode and status(name) == 'PASS':
+            mark(name, 'ERROR')
+
+    def retryable(name: str) -> bool:
+        return (
+            retry_failed > 0
+            and retries.get(name, 0) < retry_failed
+            and status(name) in ('FAIL', 'UNKNOWN')
+            # A task that produced no new work (empty diff, byte-identical retry) would
+            # only repeat itself; its marker says so and it is left for a human.
+            and not (plan / 'tasks' / name / 'noretry').exists()
+            # The retry policy found no stronger model after repeated failures on one: a
+            # split, or an explicit `retry --dwarf`, is the human's call.
+            and not (plan / 'tasks' / name / 'escalation.stop').exists()
+        )
+
+    def requeue(name: str) -> bool:
+        """Prepare a retry exactly as `retry` does, then put the task at the queue head."""
+        result = subprocess.run(['/bin/bash', script, '_prepare_retry', str(plan), name])
+        if result.returncode:
+            return False
+        retries[name] = retries.get(name, 0) + 1
+        retried.add(name)
+        if name not in pending:
+            pending.insert(0, name)
+        print('forge: retrying ' + name + ' (automatic retry ' + str(retries[name]) + '/'
+              + str(retry_failed) + ')', file=sys.stderr, flush=True)
+        return True
+
+    def echo_output(name: str) -> None:
+        # task.out is opened in append mode, so a retried task's file keeps its earlier
+        # attempts; print only what is new.
+        try:
+            data = (plan / 'tasks' / name / 'task.out').read_bytes()
+        except OSError:
+            return
+        start = offsets.get(name, 0)
+        offsets[name] = len(data)
+        print(data[start:].decode('utf-8', 'replace'), end='', flush=True)
+
+    def stop(signum: int, frame: object) -> None:
+        nonlocal interrupted
+        interrupted = True
+
+    signal.signal(signal.SIGTERM, stop)
+    signal.signal(signal.SIGINT, stop)
+    try:
+        # Failures require an explicit retry unless this run asked for automatic ones.
+        # Saved PASS results use the same fingerprint checks without paying for
+        # implementation or QA again; an INFRA stop is not a failure of the work, so it
+        # is simply dispatched again.
+        for name in pending[:]:
+            state = status(name)
+            if state == 'PASS':
+                merge(name)
+            if state == 'RUNNING':
+                mark(name, 'INTERRUPTED')
+            if state in ('FAIL', 'UNKNOWN') and retryable(name):
+                requeue(name)
+            if status(name) not in ('PENDING', 'BLOCKED', 'INFRA'):
+                pending.remove(name)
+        while pending or active:
+            if interrupted:
+                break
+            for name, (process, log) in list(active.items()):
+                if process.poll() is not None:
+                    log.close()
+                    del active[name]
+                    if os.environ.get('FORGE_OUTPUT') == 'full':
+                        echo_output(name)
+                    state = status(name)
+                    if state == 'PASS':
+                        merge(name)
+                    elif state == 'RUNNING':
+                        mark(name, 'ERROR')
+                    elif state == 'INFRA':
+                        paused = True
+                        if name not in infra_seen:
+                            infra_seen.append(name)
+                    elif state in ('FAIL', 'UNKNOWN') and retryable(name):
+                        requeue(name)
+            if paused and not active and infra_seen and infra_cycles < infra_retries:
+                # Everything running has drained. Wait out the outage once for all the
+                # tasks it stopped, then dispatch them again.
+                wait = max(infra_wait(infra_info(plan / 'tasks' / n), infra_cycles) for n in infra_seen)
+                infra_cycles += 1
+                print('forge: infrastructure failure on ' + ', '.join(infra_seen)
+                      + '; waiting ' + ('%g' % wait) + 's before resuming (no attempt spent)',
+                      file=sys.stderr, flush=True)
+                resume_at = time.time() + wait
+                while time.time() < resume_at and not interrupted:
+                    time.sleep(min(0.05, max(resume_at - time.time(), 0)))
+                if interrupted:
+                    break
+                for name in reversed(infra_seen):
+                    if name not in pending:
+                        pending.insert(0, name)
+                infra_seen.clear()
+                paused = False
+            for name in pending[:]:
+                if paused or len(active) >= capacity or interrupted:
+                    break
+                deps = [d for d in tasks[name][1].split(',') if d and d != '-']
+                if any(status(dep) != 'MERGED' for dep in deps):
+                    continue
+                if any(overlap(tasks[name][3], tasks[other][3]) for other in active):
+                    continue
+                root = plan / 'tasks' / name
+                if not (root / 'base_ref').exists():
+                    worktree = Path((plan / 'wt_root').read_text().strip()) / '_integration'
+                    base = subprocess.check_output(['git', '-C', str(worktree), 'rev-parse', 'HEAD'])
+                    (root / 'base_ref').write_bytes(base)
+                log = open(root / 'task.out', 'ab')
+                # A retried task runs with the same environment `retry` gives it, so the
+                # no-progress guard (byte-identical diff) applies under Fractal too.
+                env = (dict(os.environ, FORGE_FRACTAL_RETRY='1', FORGE_DATASET_TRIGGER='auto')
+                       if name in retried else None)
+                process = subprocess.Popen(
+                    ['/bin/bash', script, '_task', str(plan), name],
+                    stdout=log, stderr=subprocess.STDOUT, start_new_session=True, env=env,
+                )
+                active[name] = process, log
+                pending.remove(name)
+                print('forge: dispatched ' + name + ' -> ' + str(root), file=sys.stderr, flush=True)
+            if not active:
+                if paused:
+                    # Infrastructure stop: leave everything not yet run PENDING. Marking it
+                    # BLOCKED would turn "the provider was down" into a recorded failure.
+                    break
+                for name in pending:
+                    mark(name, 'BLOCKED')
+                break
+            time.sleep(0.05)
+    finally:
+        for process, log in active.values():
+            try:
+                os.killpg(process.pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+        for name, (process, log) in active.items():
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                pass
+            # Remove descendants even if the shell exited before its backend.
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+            process.wait()
+            log.close()
+            mark(name, 'INTERRUPTED')
+    if interrupted:
+        return 130
+    if paused or any(status(name) == 'INFRA' for name in tasks):
+        return INFRA_EXIT
+    return 0 if all(status(name) == 'MERGED' for name in tasks) else 5
+
+
+def main() -> None:
+    try:
+        if sys.argv[1] == 'lock':
+            _, script, command, plan, *args = sys.argv[1:]
+            # The shell inherits the lock for its entire operation, including
+            # preflight, setup, scheduling, verification and cleanup.
+            lock = open(Path(plan) / 'schedule.lock', 'a')
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            os.set_inheritable(lock.fileno(), True)
+            env = dict(os.environ, FORGE_PLAN_LOCK=plan)
+            os.execve('/bin/bash', ['/bin/bash', script, command, plan, *args], env)
+        parser = argparse.ArgumentParser(prog='forge-schedule.py')
+        parser.add_argument('script')
+        parser.add_argument('plan')
+        parser.add_argument('capacity', type=int)
+        parser.add_argument('--retry-failed', type=int, default=0, metavar='N')
+        parser.add_argument('--infra-retries', type=int, default=0, metavar='M')
+        args = parser.parse_args(sys.argv[1:])
+        sys.exit(schedule(args.script, args.plan, args.capacity, args.retry_failed,
+                          args.infra_retries))
+    except (OSError, ValueError, KeyError) as error:
+        print('forge scheduler: ' + str(error), file=sys.stderr)
+        sys.exit(3)
+
+
+if __name__ == '__main__':
+    main()

package/scripts/forge-shadow.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""A shadow review: a second, independent review of a sampled task attempt, never consumed.
+
+Nothing in forge has ever reviewed one diff twice, so how often two reviewers agree
+-- the noise in every review label -- is unknown. `plan --shadow-qa <spec>[,<spec>...]`
+turns this on for a sampled share of reviewed attempts (`--shadow-rate`, default 0.1).
+
+  forge-shadow.py run --plan P --task ID --worktree W --base B --reviewed R --attempt N --qa SPEC [--yolo]
+      Started in the background by the task, just before the primary review. Exits at once
+      unless this attempt is sampled. Otherwise it builds its own review snapshot and run
+      dir under <plan>/shadow/<id>/<attempt>-<k>/, reviews a byte-identical copy of the
+      primary's qa.input, and returns. Never exits nonzero.
+  forge-shadow.py preflight --plan P
+      Which listed shadow specs resolve here; writes <plan>/shadow_qa.ok. Never fails a run.
+
+Guarantees, because the task waits for this process before it writes any status:
+  - nothing reads the shadow's result in the run path; it changes no status, memory,
+    ledger, gate or retry, and its files never share a name or a dir with the primary's
+  - it holds no inherited file descriptor (close_fds), so no lock outlives the task
+  - it dies with the task: when its parent goes away (an interrupt, a killed scheduler,
+    Ctrl-C on a manual retry, which never reaches a background job) it stops the review
+  - it is bounded: once the task signals that the primary review finished (SIGUSR1), the
+    shadow gets as long again as the primary took, at least SHADOW_GRACE_S, then stops
+<shadow dir>/shadow.status (and a `shadow_review` dataset event) says how it ended: done,
+timeout, error, invalid (the reviewer changed its snapshot), overtime, stopped, orphaned.
+Only a `done` review counts as a second opinion.
+Forge never picks a model: the reviewer is one the user listed, preferring a different
+model family from the primary reviewer's, so two reviewers do not share one blind spot.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+SCRIPTS = Path(__file__).resolve().parent
+DISPATCH = SCRIPTS / 'forge-dispatch.sh'
+ARTIFACT = SCRIPTS / 'forge-artifact.sh'
+SHADOW_GRACE_S = float(os.environ.get('FORGE_SHADOW_GRACE_S') or 300)
+STOP_WAIT_S = 10
+PREPARE_TIMEOUT_S = 120
+
+
+def rate_of(plan: Path) -> float:
+    try:
+        rate = float((plan / 'shadow_rate').read_text().strip())
+    except (OSError, ValueError):
+        rate = 0.1
+    return min(max(rate, 0.0), 1.0)
+
+
+def sampled(plan: Path, task: str, attempt: str, rate: float) -> bool:
+    """Deterministic per attempt: a QA-only resume reaches the same answer."""
+    run = ''
+    for name in ('dataset.uid', 'run_id'):
+        try:
+            run = (plan / name).read_text().strip()
+            break
+        except OSError:
+            continue
+    digest = hashlib.sha256(('%s:%s:%s' % (run, task, attempt)).encode()).digest()
+    return int.from_bytes(digest[:8], 'big') < rate * 2 ** 64
+
+
+def family(model: str) -> str:
+    m = (model or '').lower().rsplit('/', 1)[-1]
+    if m.startswith(('claude', 'opus', 'sonnet', 'haiku', 'fable')):
+        return 'claude'
+    if m.startswith(('gpt', 'sol', 'astra', 'luna', 'terra', 'o1', 'o3', 'o4')) or 'gpt-' in m:
+        return 'gpt'
+    if m.startswith('gemini'):
+        return 'gemini'
+    if m.startswith('grok'):
+        return 'grok'
+    return m
+
+
+def resolved_model(spec: str) -> str | None:
+    """The model a qa dispatch of `spec` would use, by dispatch's own dry run."""
+    with tempfile.TemporaryDirectory(prefix='forge-shadow-resolve-') as tmp:
+        prompt = Path(tmp) / 'input'
+        prompt.write_text('Offline resolution')
+        try:
+            out = subprocess.run(['/bin/bash', str(DISPATCH), 'qa', spec, '--run-dir', tmp, '--prompt-file',
+                                  str(prompt), '--dry-run'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                                 timeout=60, text=True)
+        except (OSError, subprocess.SubprocessError):
+            return None
+    fields = dict(line.split('=', 1) for line in out.stdout.splitlines() if '=' in line)
+    return fields.get('model') or None
+
+
+def candidates(plan: Path) -> list:
+    path = plan / 'shadow_qa.ok' if (plan / 'shadow_qa.ok').exists() else plan / 'shadow_qa'
+    try:
+        text = path.read_text()
+    except OSError:
+        return []
+    return [s.strip() for s in text.replace('\n', ',').split(',') if s.strip()]
+
+
+def pick(listed: list, primary: str) -> tuple[str | None, str | None]:
+    """(spec, why): the first listed spec of another family than the primary's, else the first."""
+    if not listed:
+        return None, None
+    mine = family(resolved_model(primary) or primary)
+    for spec in listed:
+        if family(resolved_model(spec) or spec) != mine:
+            return spec, 'other_family'
+    return listed[0], 'same_family'
+
+
+def fresh_dir(base: Path, attempt: str) -> Path:
+    """A dir no earlier try used: a snapshot refuses an existing destination."""
+    base.mkdir(parents=True, exist_ok=True)
+    k = 1
+    while True:
+        candidate = base / ('%s-%d' % (attempt, k))
+        try:
+            candidate.mkdir()
+            return candidate
+        except FileExistsError:
+            k += 1
+
+
+def bash(function: str, *args) -> subprocess.CompletedProcess:
+    return subprocess.run(['/bin/bash', '-c', 'source "$1"; shift; "$@"', '_', str(ARTIFACT), function, *args],
+                          stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=PREPARE_TIMEOUT_S)
+
+
+def write(path: Path, text: str) -> None:
+    try:
+        path.write_text(text)
+    except OSError:
+        pass
+
+
+def dataset_event(out: Path, task: str, values: dict) -> None:
+    """How the shadow ended, in the run dataset. Its dispatch was captured when it exited,
+    before the snapshot was checked again, so an `invalid` verdict can only be told here."""
+    try:
+        if str(SCRIPTS) not in sys.path:
+            sys.path.insert(0, str(SCRIPTS))
+        from forge_dataset.capture import point
+        point('shadow_review', str(out), task, values)
+    except Exception:
+        pass
+
+
+class Stopped(Exception):
+    pass
+
+
+STATE = dict(child=None, primary_done=None, stop=False, started=time.time())
+
+
+def _primary_done(_signum, _frame):
+    if STATE['primary_done'] is None:
+        STATE['primary_done'] = time.time()
+
+
+def _stop(_signum, _frame):
+    # Before the review starts there is nothing to stop but this process; afterwards the
+    # watch loop stops the review itself, so the status file is always written.
+    if STATE['child'] is None:
+        raise Stopped()
+    STATE['stop'] = True
+
+
+def run(args) -> int:
+    # First, before anything slow: the task may signal that its review finished at any time,
+    # and SIGUSR1's default action would kill a launcher that is still preparing.
+    signal.signal(signal.SIGUSR1, _primary_done)
+    signal.signal(signal.SIGTERM, _stop)
+    plan = Path(args.plan)
+    parent = os.getppid()
+    status, out, spec, why = 'skipped', None, None, None
+    try:
+        if os.environ.get('FORGE_FRACTAL_RUN') or not sampled(plan, args.task, args.attempt, rate_of(plan)):
+            return 0
+        spec, why = pick(candidates(plan), args.qa)
+        if not spec:
+            return 0
+        out = fresh_dir(plan / 'shadow' / args.task, args.attempt)
+        meta = dict(task=args.task, attempt=args.attempt, reviewed=args.reviewed, primary=args.qa, spec=spec,
+                    pick=why, started=time.time())
+        write(out / 'shadow.json', json.dumps(meta, indent=1))
+        review = out / 'review-snapshot'        # review-*: the importer never walks into a snapshot
+        if bash('forge_review_snapshot', args.worktree, args.base, args.reviewed, str(review)).returncode:
+            status = 'error'
+            return 0
+        before = bash('forge_fingerprint', str(review)).stdout.strip()
+        shutil.copyfile(Path(args.tdir) / 'qa.input', out / 'qa.input')
+        env = dict(os.environ, FORGE_SHADOW='1', FORGE_SHADOW_ATTEMPT=args.attempt,
+                   FORGE_SHADOW_REVIEWED=args.reviewed, FORGE_SHADOW_FROM=args.qa)
+        argv = ['/bin/bash', str(DISPATCH), 'qa', spec, '--repo', str(review), '--run-dir', str(out),
+                '--review-base', (out / 'review.base').read_text().strip(),
+                '--ripwire-query-file', str(Path(args.tdir) / 'prompt.md'), '--prompt-file', str(out / 'qa.input'),
+                '--output', 'summary']
+        if args.yolo:
+            argv.append('--yolo')
+        with open(out / 'shadow.out', 'wb') as log:
+            STATE['child'] = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
+                                              env=env, close_fds=True)
+        status = watch(STATE['child'], parent)
+        after = bash('forge_fingerprint', str(review)).stdout.strip()
+        if status == 'done' and (not before or after != before):
+            status = 'invalid'          # the reviewer changed its snapshot: its verdict counts for nothing
+    except Stopped:
+        status = 'stopped'
+    except Exception as error:          # a shadow never fails the task
+        status = 'error'
+        if out is not None:
+            write(out / 'shadow.error', '%s: %s\n' % (type(error).__name__, error))
+    finally:
+        if out is not None:
+            write(out / 'shadow.status', status + '\n')
+            dataset_event(out, args.task, dict(status=status, attempt=args.attempt, reviewed=args.reviewed,
+                                               primary=args.qa, spec=spec, pick=why))
+    return 0
+
+
+def grace_ends() -> float | None:
+    """When the shadow must stop: as long again as the primary took, at least SHADOW_GRACE_S."""
+    done = STATE['primary_done']
+    if done is None:
+        return None
+    return done + max(SHADOW_GRACE_S, done - STATE['started'])
+
+
+def watch(child: subprocess.Popen, parent: int) -> str:
+    """Wait for the review; stop it when the task dies or stops it, or its grace runs out."""
+    while True:
+        try:
+            child.wait(timeout=1)
+            return 'done' if child.returncode == 0 else ('timeout' if child.returncode == 7 else 'error')
+        except subprocess.TimeoutExpired:
+            pass
+        ends = grace_ends()
+        if os.getppid() != parent:
+            reason = 'orphaned'         # the task is gone: an interrupt, or a killed scheduler
+        elif STATE['stop']:
+            reason = 'stopped'
+        elif ends is not None and time.time() >= ends:
+            reason = 'overtime'
+        else:
+            continue
+        child.terminate()               # dispatch's TERM trap reaps the harness's process group
+        try:
+            child.wait(timeout=STOP_WAIT_S)
+        except subprocess.TimeoutExpired:
+            child.kill()
+            child.wait()
+        return reason
+
+
+def preflight(args) -> int:
+    plan = Path(args.plan)
+    listed = [s.strip() for s in (plan / 'shadow_qa').read_text().replace('\n', ',').split(',') if s.strip()] \
+        if (plan / 'shadow_qa').exists() else []
+    ok = []
+    for spec in listed:
+        flags = ['--yolo'] if (plan / 'yolo_qa').exists() else []
+        try:
+            result = subprocess.run(['/bin/bash', str(DISPATCH), 'doctor', '--spec', spec, '--role', 'qa', *flags],
+                                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=120)
+            good = result.returncode == 0
+        except (OSError, subprocess.SubprocessError):
+            good = False
+        if good:
+            ok.append(spec)
+        else:
+            print('forge: shadow review spec %s is not usable here; it is skipped' % spec, file=sys.stderr)
+    write(plan / 'shadow_qa.ok', ','.join(ok) + '\n')
+    if listed and not ok:
+        print('forge: no shadow review spec is usable; shadow reviews are off for this run', file=sys.stderr)
+    return 0
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog='forge-shadow.py')
+    sub = ap.add_subparsers(dest='command', required=True)
+    p = sub.add_parser('run')
+    for name in ('plan', 'task', 'worktree', 'base', 'reviewed', 'attempt', 'qa', 'tdir'):
+        p.add_argument('--' + name, required=True)
+    p.add_argument('--yolo', action='store_true')
+    p = sub.add_parser('preflight')
+    p.add_argument('--plan', required=True)
+    args = ap.parse_args(argv)
+    try:
+        return run(args) if args.command == 'run' else preflight(args)
+    except Exception:
+        return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())