@giamme/forge 0.9.0 → 0.11.0

package/scripts/forge-runs.py

@@ -7,6 +7,10 @@ run dir under it, so you can see what happened without a browser and get the dir
 or open with `forge ui open`. The repo is the Git top level of the current directory unless
 `--repo` names one; `--all` lists the runs of every repo. It reads the same run summaries as
 forge-ui and never writes anything.
+
+`forge runs rm <run-dir> [--dry-run] [--yes] [--no-archive] [--worktrees] [--branches] [--json]`
+is the one thing here that changes anything: it deletes a run, after its traces are in the dataset
+and an archive exists (scripts/forge-runs-rm.py).
 """
 from __future__ import annotations
 
@@ -86,8 +90,22 @@ def line(run: dict, now: float, with_repo: bool = False) -> str:
     return f"{run['run_id'] or run['key']}  {run['kind']}{repo}  {age(run['updated'], now)}  {counts}{checks}  {goal}\n    {run['path']}"
 
 
+def remove(argv) -> int:
+    """`forge runs rm`: loaded on demand, so listing never pays for the dataset package."""
+    import importlib.util
+    spec = importlib.util.spec_from_file_location('forge_runs_rm', Path(__file__).resolve().with_name('forge-runs-rm.py'))
+    module = importlib.util.module_from_spec(spec)
+    sys.modules['forge_runs_rm'] = module
+    spec.loader.exec_module(module)
+    return module.main(argv)
+
+
 def main(argv=None) -> int:
-    parser = argparse.ArgumentParser(prog='forge runs', description="This repo's recent forge runs, newest first. Read-only.")
+    argv = sys.argv[1:] if argv is None else list(argv)
+    if argv[:1] == ['rm']:
+        return remove(argv[1:])
+    parser = argparse.ArgumentParser(prog='forge runs', description="This repo's recent forge runs, newest first. Listing is read-only.",
+                                     epilog='Delete a run, keeping its traces: forge runs rm <run-dir> (forge runs rm --help)')
     which = parser.add_mutually_exclusive_group()
     which.add_argument('--repo', type=Path, metavar='DIR', help='the repo whose runs to list (default: the Git top level of the current directory)')
     which.add_argument('--all', action='store_true', help='list the runs of every repo')

package/scripts/forge-runtime.py

@@ -175,6 +175,43 @@ def extract(log: str, harness: str) -> tuple[str | None, dict[str, int | None]]:
     return final, usage
 
 
+def session_id(log: str, harness: str) -> str | None:
+    """The harness session this dispatch ran in, or None.
+
+    claude and openclaude name it session_id on the stream-json init line and
+    again on the result line. The init line wins. The result line is used when
+    there is no init line, or that line has no session_id. A session_id that is
+    present but not a string is not a session, and the result line does not
+    replace it. codex names it thread_id on the first thread.started line.
+    Any other harness, a missing id, or an id that is not a plain string is None.
+    """
+    if harness not in ('claude', 'openclaude', 'codex'):
+        return None
+    saw_init = False
+    result_id = None
+    for line in log.splitlines():
+        try:
+            event = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if harness == 'codex':
+            if event.get('type') == 'thread.started':
+                value = event.get('thread_id')
+                return value if type(value) is str else None
+            continue
+        kind = event.get('type')
+        if not saw_init and kind == 'system' and event.get('subtype') == 'init':
+            saw_init = True
+            if 'session_id' in event:
+                value = event.get('session_id')
+                return value if type(value) is str else None
+        elif kind == 'result' and 'session_id' in event:
+            result_id = event.get('session_id')
+    return result_id if type(result_id) is str else None
+
+
 def finish(attempt: str, root: str, role: str, harness: str, rc: str) -> None:
     try:
         _finish(Path(attempt), Path(root), role, harness, rc)
@@ -189,6 +226,7 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
     structured = harness in STRUCTURED
     text = log.read_text(errors='replace') if structured and log.exists() else ''
     final, usage = extract(text, harness)
+    found = session_id(text, harness)
     if not last.exists() or not last.stat().st_size:
         if final is not None:
             last.write_text(final + '\n')
@@ -201,6 +239,7 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
     data.update(
         harness=harness,
         prompt_bytes=prompt.stat().st_size if prompt_ready else None,
+        session_id=found,
         **usage,
     )
     infra = root / (role + '.infra')
@@ -209,7 +248,14 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
         for line in infra.read_text(errors='replace').splitlines():
             if line.startswith('class='):
                 data['infra_class'] = line[len('class='):].strip() or None
-    for suffix in ('prompt', 'last', 'log', 'resolved', 'cmd', 'timedout', 'infra', 'orphans'):
+    # Drop a stale id first. A dispatch that published none must not leave the
+    # previous one for a later round to continue.
+    session_path = root / (role + '.session')
+    session_path.unlink(missing_ok=True)
+    if found is not None:
+        session_path.write_text(found + '\n')
+    for suffix in ('prompt', 'last', 'log', 'resolved', 'cmd', 'timedout', 'infra', 'orphans',
+                   'session', 'continue'):
         if suffix == 'prompt' and not prompt_ready:
             continue
         source = root / (role + '.' + suffix)

package/scripts/forge-schedule.py

@@ -10,17 +10,26 @@ Two opt-in behaviours live here, both off by default so a plain `run` is unchang
   network, empty output) up to M times instead of stopping. Without it the scheduler
   stops dispatching, lets running tasks drain and exits 8: nothing was the task's fault
   and no attempt was spent, so the next `run` simply resumes.
+
+Both numbers are per invocation, but while the scheduler runs they are mirrored into the
+plan dir (retry_budget, infra_budget, tasks/<id>/retry_budget) and re-read at every
+decision, so `forge control` can change them mid-run. See "live control files" below.
 """
 from __future__ import annotations
 
 import argparse
+import contextlib
+import errno
 import fcntl
+import json
 import os
 from pathlib import Path
 import posixpath
 import signal
+import stat
 import subprocess
 import sys
+import tempfile
 import time
 
 
@@ -80,6 +89,157 @@ def infra_wait(info: dict, cycle: int) -> float:
     return min(base * (2 ** cycle), max(base, 1800.0))
 
 
+# --- live control files ------------------------------------------------------------------
+# Created at start and removed at exit, so a later `run` never inherits a budget it did not
+# ask for. Writers replace files atomically, so readers need no lock; the control lock only
+# orders a lifecycle write against `forge control`'s liveness check. It is never held across
+# a model dispatch and never taken together with schedule.lock.
+CONTROL_LOCK_WAIT = 5.0   # seconds; a busy lock must never fail a run
+SWEEP_EVERY = 1.0         # seconds between looks at tasks that already ended FAIL/UNKNOWN
+
+
+def _warn(message: str) -> None:
+    print('forge: ' + message, file=sys.stderr, flush=True)
+
+
+def _task_id_ok(name: str) -> bool:
+    return bool(name) and name not in ('.', '..') and '/' not in name and '\0' not in name
+
+
+def _resolve(plan: object, *parts: str) -> str:
+    """plan/<parts> if every directory on the way is a real one (no symlink) inside the
+    canonical plan dir and an existing target is a regular file. OSError otherwise."""
+    if not all(_task_id_ok(part) for part in parts):
+        raise OSError(errno.EINVAL, 'invalid path component', '/'.join(parts))
+    path = os.path.realpath(plan)
+    for part in parts[:-1]:
+        path = os.path.join(path, part)
+        if not stat.S_ISDIR(os.lstat(path).st_mode):
+            raise OSError(errno.ENOTDIR, 'not a real directory', path)
+    path = os.path.join(path, parts[-1])
+    try:
+        if not stat.S_ISREG(os.lstat(path).st_mode):
+            raise OSError(errno.EPERM, 'not a regular file', path)
+    except FileNotFoundError:
+        pass
+    return path
+
+
+def _write_atomic(plan: object, text: str, *parts: str) -> bool:
+    """Temp file in the same directory, then rename. A failure warns and returns False."""
+    tmp = None
+    try:
+        path = _resolve(plan, *parts)
+        fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix='.' + parts[-1] + '.')
+        with os.fdopen(fd, 'w') as handle:
+            os.fchmod(handle.fileno(), 0o644)
+            handle.write(text)
+        os.replace(tmp, path)
+        return True
+    except OSError as error:
+        _warn('could not write ' + '/'.join(parts) + ': ' + str(error))
+        if tmp:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp)
+        return False
+
+
+def _remove(plan: object, *parts: str) -> None:
+    try:
+        os.unlink(_resolve(plan, *parts))
+    except FileNotFoundError:
+        pass
+    except OSError as error:
+        _warn('could not remove ' + '/'.join(parts) + ': ' + str(error))
+
+
+def _read_budget(plan: object, *parts: str) -> int | None:
+    """A non-negative integer from a control file; None when it is missing, empty, a
+    symlink or anything else, so the caller falls through to its next source."""
+    try:
+        fd = os.open(_resolve(plan, *parts), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+        try:
+            data = os.read(fd, 65)
+        finally:
+            os.close(fd)
+        text = data.decode('ascii').strip()
+    except (OSError, UnicodeDecodeError):
+        return None
+    return int(text) if len(data) <= 64 and text.isdigit() else None
+
+
+@contextlib.contextmanager
+def _control_lock(plan: object):
+    """flock on control.lock, retried for CONTROL_LOCK_WAIT seconds. On timeout or any
+    error: warn and proceed without it (yielding False), never fail the run."""
+    fd = None
+    try:
+        fd = os.open(_resolve(plan, 'control.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o644)
+        deadline = time.monotonic() + CONTROL_LOCK_WAIT
+        while True:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() >= deadline:
+                    raise OSError(errno.EWOULDBLOCK, 'busy for ' + ('%g' % CONTROL_LOCK_WAIT) + 's')
+                time.sleep(0.05)
+    except OSError as error:
+        _warn('control.lock not taken, continuing without it: ' + str(error))
+        if fd is not None:
+            os.close(fd)
+            fd = None
+    try:
+        yield fd is not None
+    finally:
+        if fd is not None:
+            os.close(fd)
+
+
+def _task_dirs(plan: object, names: object) -> list[str]:
+    """The planned tasks plus any other task dir, since `forge control` accepts an id
+    that is not in tasks.tsv and its file must not outlive the invocation either."""
+    found = set(names)
+    try:
+        with os.scandir(os.path.join(os.path.realpath(plan), 'tasks')) as entries:
+            found.update(e.name for e in entries if _task_id_ok(e.name) and e.is_dir(follow_symlinks=False))
+    except OSError:
+        pass
+    return sorted(found)
+
+
+def _pinned(plan: object, name: str) -> bool:
+    try:
+        return os.path.lexists(_resolve(plan, 'tasks', name, 'escalation.pin'))
+    except OSError:
+        return False
+
+
+def live_start(plan: object, names: object, retry_failed: int, infra_retries: int) -> None:
+    """Publish the flags and that this scheduler is live. Stale per-task overrides and
+    usage counters from an earlier invocation go first."""
+    with _control_lock(plan):
+        for name in _task_dirs(plan, names):
+            _remove(plan, 'tasks', name, 'retry_budget')
+            _remove(plan, 'tasks', name, 'retries.used')
+        _write_atomic(plan, str(retry_failed), 'retry_budget')
+        _write_atomic(plan, str(infra_retries), 'infra_budget')
+        _write_atomic(plan, json.dumps({'pid': os.getpid(), 'started': time.time()}), 'scheduler.live')
+
+
+def live_stop(plan: object, names: object) -> None:
+    """Withdraw liveness first, then every budget. retries.used stays for the UI."""
+    try:
+        with _control_lock(plan):
+            _remove(plan, 'scheduler.live')
+            _remove(plan, 'retry_budget')
+            _remove(plan, 'infra_budget')
+            for name in _task_dirs(plan, names):
+                _remove(plan, 'tasks', name, 'retry_budget')
+    except Exception as error:
+        _warn('could not withdraw the live control files: ' + str(error))
+
+
 def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
              infra_retries: int = 0) -> int:
     if capacity <= 0:
@@ -97,16 +257,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
     if len(tasks) != len(rows):
         raise ValueError('duplicate task IDs')
     for name in tasks:
-        if not name or name in ('.', '..') or '/' in name:
+        if not _task_id_ok(name):
             raise ValueError('invalid task ID: ' + name)
         (plan / 'tasks' / name).mkdir(parents=True, exist_ok=True)
     active = {}
     pending = list(tasks)
     interrupted = False
     retries = {}       # automatic reviewer-driven retries used per task, this run
+    refused = {}       # task -> budget at the time its retry could not be prepared
     retried = set()    # tasks whose next dispatch is a retry (FORGE_FRACTAL_RETRY=1)
     offsets = {}       # how much of each task.out was already echoed under --output full
     infra_seen = []    # tasks stopped by an infrastructure failure, awaiting resume
+    held = set()       # INFRA tasks not resumed: their pin could not be applied
     infra_cycles = 0   # how many times this run already waited out an infrastructure pause
     paused = False     # an infrastructure failure was seen: dispatch nothing new
     resume_at = 0.0
@@ -127,30 +289,61 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
         if result.returncode and status(name) == 'PASS':
             mark(name, 'ERROR')
 
+    def budget(name: str) -> int:
+        """Automatic retries this task may use. Read at every decision, never cached: the
+        task's own file, then the run-wide file, then the flag."""
+        for parts in (('tasks', name, 'retry_budget'), ('retry_budget',)):
+            value = _read_budget(plan, *parts)
+            if value is not None:
+                return value
+        return retry_failed
+
+    def infra_budget() -> int:
+        value = _read_budget(plan, 'infra_budget')
+        return infra_retries if value is None else value
+
     def retryable(name: str) -> bool:
+        root = plan / 'tasks' / name
         return (
-            retry_failed > 0
-            and retries.get(name, 0) < retry_failed
-            and status(name) in ('FAIL', 'UNKNOWN')
+            status(name) in ('FAIL', 'UNKNOWN')
+            and retries.get(name, 0) < budget(name)
             # A task that produced no new work (empty diff, byte-identical retry) would
             # only repeat itself; its marker says so and it is left for a human.
-            and not (plan / 'tasks' / name / 'noretry').exists()
+            and not (root / 'noretry').exists()
             # The retry policy found no stronger model after repeated failures on one: a
-            # split, or an explicit `retry --dwarf`, is the human's call.
-            and not (plan / 'tasks' / name / 'escalation.stop').exists()
+            # split, or an explicit `retry --dwarf`, is the human's call. A pin is exactly
+            # that call, made ahead of time: the next full retry applies it.
+            and (not (root / 'escalation.stop').exists() or _pinned(plan, name))
         )
 
     def requeue(name: str) -> bool:
         """Prepare a retry exactly as `retry` does, then put the task at the queue head."""
         result = subprocess.run(['/bin/bash', script, '_prepare_retry', str(plan), name])
         if result.returncode:
+            refused[name] = budget(name)
             return False
+        refused.pop(name, None)
         retries[name] = retries.get(name, 0) + 1
+        _write_atomic(plan, str(retries[name]), 'tasks', name, 'retries.used')
         retried.add(name)
         if name not in pending:
             pending.insert(0, name)
         print('forge: retrying ' + name + ' (automatic retry ' + str(retries[name]) + '/'
-              + str(retry_failed) + ')', file=sys.stderr, flush=True)
+              + str(budget(name)) + ')', file=sys.stderr, flush=True)
+        return True
+
+    def resume(name: str) -> bool:
+        """Prepare an INFRA task's next entry. Not a retry, so nothing is counted, but a
+        pending pin applies when that entry is a full implementer attempt. False when a
+        claimed pin could not be put in place: the task then stays INFRA, as `retry` would."""
+        if not _pinned(plan, name):
+            return True
+        result = subprocess.run(['/bin/bash', script, '_prepare_retry', str(plan), name, '--resume'])
+        if result.returncode:
+            held.add(name)
+            print('forge: ' + name + ' is not resumed: its escalation pin could not be applied',
+                  file=sys.stderr, flush=True)
+            return False
         return True
 
     def echo_output(name: str) -> None:
@@ -171,6 +364,7 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
     signal.signal(signal.SIGTERM, stop)
     signal.signal(signal.SIGINT, stop)
     try:
+        live_start(plan, tasks, retry_failed, infra_retries)
         # Failures require an explicit retry unless this run asked for automatic ones.
         # Saved PASS results use the same fingerprint checks without paying for
         # implementation or QA again; an INFRA stop is not a failure of the work, so it
@@ -183,13 +377,19 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                 mark(name, 'INTERRUPTED')
             if state in ('FAIL', 'UNKNOWN') and retryable(name):
                 requeue(name)
+            if state == 'INFRA' and not resume(name):
+                pending.remove(name)
+                continue
             if status(name) not in ('PENDING', 'BLOCKED', 'INFRA'):
                 pending.remove(name)
+        swept = time.monotonic()
         while pending or active:
             if interrupted:
                 break
+            ended = set()
             for name, (process, log) in list(active.items()):
                 if process.poll() is not None:
+                    ended.add(name)
                     log.close()
                     del active[name]
                     if os.environ.get('FORGE_OUTPUT') == 'full':
@@ -205,7 +405,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                             infra_seen.append(name)
                     elif state in ('FAIL', 'UNKNOWN') and retryable(name):
                         requeue(name)
-            if paused and not active and infra_seen and infra_cycles < infra_retries:
+            if time.monotonic() - swept >= SWEEP_EVERY:
+                # A human can raise a budget after a task already ended FAIL while others
+                # still run. The initial pass's predicate, for tasks nothing else will look
+                # at again; one whose retry could not be prepared waits for a new budget.
+                swept = time.monotonic()
+                for name in tasks:
+                    if name in pending or name in active or name in ended:
+                        continue
+                    if status(name) in ('FAIL', 'UNKNOWN') and retryable(name) \
+                            and refused.get(name) != budget(name):
+                        requeue(name)
+            if paused and not active and infra_seen and infra_cycles < infra_budget():
                 # Everything running has drained. Wait out the outage once for all the
                 # tasks it stopped, then dispatch them again.
                 wait = max(infra_wait(infra_info(plan / 'tasks' / n), infra_cycles) for n in infra_seen)
@@ -218,8 +429,10 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                     time.sleep(min(0.05, max(resume_at - time.time(), 0)))
                 if interrupted:
                     break
+                # A pin set before or during the wait applies to the entries that run the
+                # implementer again; a QA-stage stop keeps it for the next full retry.
                 for name in reversed(infra_seen):
-                    if name not in pending:
+                    if name not in pending and resume(name):
                         pending.insert(0, name)
                 infra_seen.clear()
                 paused = False
@@ -249,15 +462,18 @@ def schedule(script: str, plan: str, capacity: int, retry_failed: int = 0,
                 pending.remove(name)
                 print('forge: dispatched ' + name + ' -> ' + str(root), file=sys.stderr, flush=True)
             if not active:
-                if paused:
+                if paused or held:
                     # Infrastructure stop: leave everything not yet run PENDING. Marking it
                     # BLOCKED would turn "the provider was down" into a recorded failure.
+                    # A task held back by its pin is still an infrastructure stop.
                     break
                 for name in pending:
                     mark(name, 'BLOCKED')
                 break
             time.sleep(0.05)
     finally:
+        # Not live from here on: a late `forge control` must not edit budgets nobody reads.
+        live_stop(plan, tasks)
         for process, log in active.values():
             try:
                 os.killpg(process.pid, signal.SIGTERM)