@giamme/forge 0.10.0 → 0.11.1

package/scripts/forge-prompt.py

@@ -5,20 +5,72 @@ from __future__ import annotations
 import argparse
 from pathlib import Path
 
+DIFF_INLINE_CAP = 100 * 1024
+REPLY_CAP = 8 * 1024
+
+# What a reviewer is asked to do with the previous round's report. The labelled-line rule
+# matters: the severity gate, `forge issues` and forge-ui count every `- [Px][...]` line as open.
+QA_ROUND_NOTE = """\
+Check each finding above first: is it fixed, still present, or validly rejected by the
+implementer (see their reply below)? Report a finding that is still present again, in the
+usual labelled format with its severity. Mention a fixed or rejected finding only in a plain
+sentence, never as a `- [Px][CONFIRMED]` or `- [Px][PLAUSIBLE]` line: every labelled line is
+counted as an open finding. This round's threshold, in the contract below, governs; it may
+differ from the last round's. Then review the full task diff below: the changes since that
+round are where new problems are most likely."""
+
+
+def read(path: str | None, errors: str = 'strict') -> str:
+    if path and Path(path).is_file():
+        with Path(path).open(newline="", errors=errors) as source:
+            return source.read()
+    return ''
+
+
+def carried(text: str) -> str:
+    # The dataset and forge-ui take the LAST ```diff fence of a QA prompt as the reviewed
+    # diff, so nothing carried from an earlier round may open one.
+    return text.replace('```diff', '```text')
+
+
+def qa_round(findings: str, diff: str | None, span: str | None, stat: str | None, reply: str | None) -> str:
+    parts = ['## Findings from the previous review round\n' + carried(findings).rstrip('\n') + '\n\n' + QA_ROUND_NOTE]
+    # A git diff carries raw source bytes, so a file that is not UTF-8 must not abort the prompt.
+    patch = read(diff, 'replace')
+    if patch:
+        label = ' (' + span.strip() + ')' if span and span.strip() else ''
+        if len(patch.encode()) <= DIFF_INLINE_CAP:
+            body = '```patch\n' + patch.rstrip('\n') + '\n```'
+        else:
+            body = ('The diff since that review is too large to inline; its stat:\n\n```text\n'
+                    + read(stat, 'replace').rstrip('\n') + '\n```')
+        parts.append('### Changes since that review' + label + '\n' + body)
+    said = read(reply, 'replace')
+    if said.strip():
+        raw = said.encode()
+        if len(raw) > REPLY_CAP:
+            said = raw[:REPLY_CAP].decode(errors='ignore') + '\n[... the reply continues; cut here ...]\n'
+        parts.append("### The implementer's reply (its own account; verify every claim against the code)\n"
+                     + carried(said).rstrip('\n'))
+    return '\n\n'.join(parts) + '\n'
+
 
 def context(
     requirements: str, approach: str | None = None, capsule: str | None = None,
     retry: str | None = None, goal: str | None = None, memories: tuple[str, ...] = (),
+    role: str = 'dwarf', retry_diff: str | None = None, retry_range: str | None = None,
+    retry_stat: str | None = None, reply: str | None = None,
 ) -> str:
     chunks = []
     for title, path in [('Ownership and baseline', capsule), ('Overall goal', goal),
                         ('Complete implementation requirements', requirements),
                         ('Intended approach', approach),
                         ('Your previous attempt was reviewed and rejected', retry)]:
-        if path and Path(path).is_file():
-            with Path(path).open(newline="") as source:
-                data = source.read()
-            if data:
+        data = read(path) if path and Path(path).is_file() else ''
+        if data:
+            if role == 'qa' and path == retry:
+                chunks.append(qa_round(data, retry_diff, retry_range, retry_stat, reply).rstrip('\n'))
+            else:
                 chunks.append('## ' + title + '\n' + data)
     # Memory injection emits headings and prose around bullet facts. Union only
     # exact fact lines, preserving distinct facts even when they share a key.
@@ -40,6 +92,11 @@ if __name__ == '__main__':
     for key in ('requirements', 'approach', 'capsule', 'retry', 'goal'):
         parser.add_argument('--' + key, required=key == 'requirements')
     parser.add_argument('--memory', action='append', default=[])
+    parser.add_argument('--role', choices=('dwarf', 'qa'), default='dwarf')
+    parser.add_argument('--retry-diff')
+    parser.add_argument('--retry-range')
+    parser.add_argument('--retry-stat')
+    parser.add_argument('--reply')
     args = vars(parser.parse_args())
     args['memories'] = args.pop('memory')
     print(context(**args), end='')

package/scripts/forge-route.py

@@ -16,7 +16,7 @@ bias (inverse-propensity weighting needs the probability each choice had).
       rewritten) and marks a changed pick in tasks/<id>/route.explored for the table. A row
       with no neighbour is not explored and records nothing.
 
-  forge-route.py escalation --plan P --task ID [--check]
+  forge-route.py escalation --plan P --task ID [--check] [--qa-only-next yes|no]
       The retry policy: after <plan>/escalate_after (default 3; 0 = off) failed attempts on
       one spec, the next attempt goes to the next tier up of the user's pools instead of
       repeating the same dispatch. Prints that spec when one is due. When none exists, writes
@@ -25,11 +25,21 @@ bias (inverse-propensity weighting needs the probability each choice had).
       the streak no longer holds. A pending tasks/<id>/escalation.pin (a human's one-shot
       choice, applied by prepare_retry) is what `--check` reports as the next model instead
       (escalation.next = its spec), whatever escalate_after says, unless the next retry is a
-      QA-only resume or the plan is Fractal (pins never apply there). Nothing has applied the
-      pin yet, so the policy's own stop and split markers stay as the policy decided them
-      beneath it: they are what holds again if the pin is withdrawn.
-      The marker writes take <plan>/control.lock and re-read escalate_after inside it: a
-      policy decision made before a human switched the policy off writes nothing.
+      QA-only resume or the plan is Fractal (pins never apply there). `--qa-only-next` is
+      that fact as the caller measured it; without the option, a resume marker or an
+      unreviewed commit is taken as QA-only. Nothing has applied the pin yet, so the policy's
+      own stop and split markers stay as the policy decided them beneath it: they are what
+      holds again if the pin is withdrawn. The marker writes take <plan>/control.lock and
+      re-read escalate_after inside it: a policy decision made before a human switched the
+      policy off writes nothing.
+
+  forge-route.py record --plan P --task ID (--probe | --line TEXT)
+      Internal. Probe or append tasks/<id>/escalations.tsv through forge-control.py, which
+      walks from the plan one directory at a time and refuses a symlink at any component.
+      `--probe` exits 0 when every component below the plan is a real directory and the file
+      is absent or a regular file. `--line` appends one record under those same conditions.
+      A missing parent exits 1 and writes nothing: the walk has no directory descriptor then,
+      and opening the leaf would create it in the process's current directory.
 
 The propensity of the chosen spec is (1-E)*[chosen == SPEC] + E/2. Forge never picks a
 model the user did not name: both helpers read only the pools `plan` stored in
@@ -389,11 +399,10 @@ def _pin(tdir: Path):
 
 
 def _qa_next(tdir: Path) -> bool:
-    """True when the task's next retry only re-runs QA, so it cannot apply a pin.
+    """Fallback when `--qa-only-next` is absent: a resume marker, or an unreviewed commit.
 
-    That is a `resume` marker (a QA-stage infrastructure stop), or a verdict-less review of
-    committed work: prepare_retry turns that into the marker, but it has not run yet when
-    `--check` does, right after the attempt.
+    This does not look at the worktree. Callers that measured it (qa_only_next) pass
+    `--qa-only-next` instead, so a marker the worktree no longer matches is a full retry.
     """
     if (tdir / 'resume').exists():
         return True
@@ -455,8 +464,10 @@ def escalation(args) -> int:
     idle = after <= 0 or fractal or (tdir / 'resume').exists()
     # A pin is a human's choice and outranks the policy, so `--check` reports it as the next
     # model whatever escalate_after says; prepare_retry applies it and claims the file. The
-    # policy is still decided underneath it and its markers kept current.
-    pin = _pin(tdir) if args.check and not fractal and not _qa_next(tdir) else None
+    # policy is still decided underneath it and its markers kept current. The policy's own
+    # idle test above is separate and stays on the raw resume marker.
+    qa_only = _qa_next(tdir) if args.qa_only_next is None else args.qa_only_next == 'yes'
+    pin = _pin(tdir) if args.check and not fractal and not qa_only else None
     clear, writes, shown, rc = (MARKERS if after <= 0 else ()), (), '', 0
     if not idle:
         clear, writes, shown, rc = _decide(plan, tdir, args, after)
@@ -474,6 +485,45 @@ def escalation(args) -> int:
     return rc
 
 
+def _load_control():
+    """forge-control.py, by path: probe and append_line refuse a symlink at any component."""
+    import importlib.util
+    spec = importlib.util.spec_from_file_location(
+        'forge_control', Path(__file__).resolve().with_name('forge-control.py'))
+    module = importlib.util.module_from_spec(spec)
+    sys.modules['forge_control'] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def record_escalation(args) -> int:
+    """Probe or append tasks/<id>/escalations.tsv. 0 when that is safe; 1 when it is not.
+
+    A missing parent is a refusal on both paths. probe() reports that as the file being
+    absent, so the check has to look at the walk itself. append_line refuses the same case
+    before it opens the leaf: with no directory descriptor that open would use the process
+    cwd, including when the task directory disappears between this check and the append.
+    """
+    if not args.task or args.task in ('.', '..') or '/' in args.task or '\0' in args.task:
+        return 1
+    try:
+        control = _load_control()
+        plan = os.path.realpath(args.plan)
+        rel = 'tasks/%s/escalations.tsv' % args.task
+        parent, _leaf = control._open_parent(plan, rel)
+        if parent is None:
+            return 1
+        os.close(parent)
+        if args.probe:
+            control.probe(plan, rel)
+        else:
+            line = args.line if args.line.endswith('\n') else args.line + '\n'
+            control.append_line(plan, rel, line)
+    except Exception:
+        return 1
+    return 0
+
+
 def main(argv=None) -> int:
     ap = argparse.ArgumentParser(prog='forge-route.py')
     sub = ap.add_subparsers(dest='command', required=True)
@@ -484,6 +534,14 @@ def main(argv=None) -> int:
     p.add_argument('--plan', required=True)
     p.add_argument('--task', required=True)
     p.add_argument('--check', action='store_true')
+    p.add_argument('--qa-only-next', choices=('yes', 'no'), default=None,
+                   help='whether the next entry only re-runs QA, as the caller measured it')
+    p = sub.add_parser('record', help=argparse.SUPPRESS)
+    p.add_argument('--plan', required=True)
+    p.add_argument('--task', required=True)
+    mode = p.add_mutually_exclusive_group(required=True)
+    mode.add_argument('--probe', action='store_true')
+    mode.add_argument('--line')
     args = ap.parse_args(argv)
     if args.command == 'explore':
         try:
@@ -491,6 +549,8 @@ def main(argv=None) -> int:
         except Exception:
             print(args.default)
         return 0
+    if args.command == 'record':
+        return record_escalation(args)
     try:
         return escalation(args)
     except Exception:

package/scripts/forge-runtime.py

@@ -175,6 +175,43 @@ def extract(log: str, harness: str) -> tuple[str | None, dict[str, int | None]]:
     return final, usage
 
 
+def session_id(log: str, harness: str) -> str | None:
+    """The harness session this dispatch ran in, or None.
+
+    claude and openclaude name it session_id on the stream-json init line and
+    again on the result line. The init line wins. The result line is used when
+    there is no init line, or that line has no session_id. A session_id that is
+    present but not a string is not a session, and the result line does not
+    replace it. codex names it thread_id on the first thread.started line.
+    Any other harness, a missing id, or an id that is not a plain string is None.
+    """
+    if harness not in ('claude', 'openclaude', 'codex'):
+        return None
+    saw_init = False
+    result_id = None
+    for line in log.splitlines():
+        try:
+            event = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if harness == 'codex':
+            if event.get('type') == 'thread.started':
+                value = event.get('thread_id')
+                return value if type(value) is str else None
+            continue
+        kind = event.get('type')
+        if not saw_init and kind == 'system' and event.get('subtype') == 'init':
+            saw_init = True
+            if 'session_id' in event:
+                value = event.get('session_id')
+                return value if type(value) is str else None
+        elif kind == 'result' and 'session_id' in event:
+            result_id = event.get('session_id')
+    return result_id if type(result_id) is str else None
+
+
 def finish(attempt: str, root: str, role: str, harness: str, rc: str) -> None:
     try:
         _finish(Path(attempt), Path(root), role, harness, rc)
@@ -189,6 +226,7 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
     structured = harness in STRUCTURED
     text = log.read_text(errors='replace') if structured and log.exists() else ''
     final, usage = extract(text, harness)
+    found = session_id(text, harness)
     if not last.exists() or not last.stat().st_size:
         if final is not None:
             last.write_text(final + '\n')
@@ -201,6 +239,7 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
     data.update(
         harness=harness,
         prompt_bytes=prompt.stat().st_size if prompt_ready else None,
+        session_id=found,
         **usage,
     )
     infra = root / (role + '.infra')
@@ -209,7 +248,14 @@ def _finish(attempt: Path, root: Path, role: str, harness: str, rc: str) -> None
         for line in infra.read_text(errors='replace').splitlines():
             if line.startswith('class='):
                 data['infra_class'] = line[len('class='):].strip() or None
-    for suffix in ('prompt', 'last', 'log', 'resolved', 'cmd', 'timedout', 'infra', 'orphans'):
+    # Drop a stale id first. A dispatch that published none must not leave the
+    # previous one for a later round to continue.
+    session_path = root / (role + '.session')
+    session_path.unlink(missing_ok=True)
+    if found is not None:
+        session_path.write_text(found + '\n')
+    for suffix in ('prompt', 'last', 'log', 'resolved', 'cmd', 'timedout', 'infra', 'orphans',
+                   'session', 'continue'):
         if suffix == 'prompt' and not prompt_ready:
             continue
         source = root / (role + '.' + suffix)

package/scripts/forge-ui.py

@@ -355,12 +355,16 @@ def cmd_open(args):
                 # the record and the live answer must both name this code and the mode asked for
                 if (server.get('build') == build and answer.get('build') == build and server.get('registry') == registry
                         and bool(server.get('read_only')) == read_only and is_read_only(answer) == read_only):
-                    # Reused, it would run this shell's writes with another registry and dataset, and a
-                    # restart could cut off a write in flight: say so instead, and how to restart it.
+                    # Reused, its CLIs would run with another registry and dataset. A server that takes
+                    # writes may be in the middle of one, which a restart could cut off: say so instead,
+                    # and how to restart it. A read-only one has none to cut off: replace it.
                     change = config_change(server.get('cli_env'), cli_env)
-                    if change:
+                    if change and read_only:
+                        terminate(server['pid'])
+                        server = None
+                    elif change:
                         _say(f'error: the running forge-ui (pid {server["pid"]}) has another forge configuration '
-                             f'for its writes: {change}. Run `forge ui stop`, then open again.')
+                             f'for its CLIs: {change}. Run `forge ui stop`, then open again.')
                         return 5
                 else:
                     terminate(server['pid'])

package/scripts/forge_dataset/__init__.py

@@ -94,8 +94,13 @@ def data_home() -> Path:
     explicit = os.environ.get('FORGE_DATA_HOME')
     if explicit:
         return Path(explicit).expanduser()
-    base = os.environ.get('XDG_DATA_HOME') or str(Path.home() / '.local' / 'share')
-    return Path(base).expanduser() / 'forge'
+    # An empty or relative XDG_DATA_HOME counts as unset, as the XDG spec says; taken
+    # literally it named a directory below the current one, usually the user's repo.
+    value = os.environ.get('XDG_DATA_HOME') or ''
+    base = Path(value).expanduser() if value else None
+    if base is None or not base.is_absolute():
+        base = Path.home() / '.local' / 'share'
+    return base / 'forge'
 
 
 def dataset_dir() -> Path:

package/scripts/forge_dataset/cli.py

@@ -1,6 +1,6 @@
 """`forge dataset ...`: inspect the run dataset, import old runs, and the shell's capture hooks.
 
-User-facing: status, dump, import, export, outcomes. Internal (called by forge's shell scripts, fail-open,
+User-facing: status, dump, import, export, outcomes, routing, reviewers, forecast. Internal (called by forge's shell scripts, fail-open,
 silent): event, invocation, plan.
 """
 from __future__ import annotations
@@ -28,6 +28,24 @@ def _human(n: int) -> str:
     return str(n)
 
 
+def _continued_sessions(db) -> dict:
+    """How many dispatches asked to continue a reviewer session.
+
+    Counted from the dispatch events themselves. A shadow review is not required, and a
+    continue that failed still counts: the file was written because continuing was requested.
+    """
+    continued = failed = 0
+    for (raw,) in db.execute(
+            "SELECT data FROM events WHERE kind = 'dispatch' AND data LIKE ?",
+            ('%"session_continue": "%',)):
+        value = json.loads(raw).get('session_continue')
+        if value == 'continued':
+            continued += 1
+        elif value == 'failed':
+            failed += 1
+    return dict(continued_reviews=continued, failed_continues=failed)
+
+
 def status(args) -> int:
     from . import checks
     directory = dataset_dir()
@@ -43,6 +61,7 @@ def status(args) -> int:
             last = db.execute('SELECT MAX(ts) FROM events').fetchone()[0]
             summary['last_event'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last)) if last else None
             summary['checks'] = checks.run(db, verbose=args.verbose)
+            summary.update(_continued_sessions(db))
             if summary['events'].get('dispatch'):
                 from . import export
                 if db.execute("SELECT 1 FROM events WHERE kind = 'dispatch' AND data LIKE '%qa_shadow%' LIMIT 1").fetchone():
@@ -77,6 +96,9 @@ def status(args) -> int:
                   % (shadow['shadow_reviews'], shadow['pairs'], shadow['agree'], rate))
             for pair, n in shadow['by_pair'].items():
                 print('          %s: %d of %d agree' % (pair, n['agree'], n['pairs']))
+        if summary.get('continued_reviews') or summary.get('failed_continues'):
+            print('  continued reviews: %d' % summary['continued_reviews'])
+            print('  failed continues: %d' % summary['failed_continues'])
     if summary['legacy_archives']:
         print('  legacy: %d archived run dir(s) kept in %s (`forge dataset import %s` re-reads them;'
               ' events already stored are skipped)' % (summary['legacy_archives'], legacy, legacy))
@@ -163,6 +185,16 @@ def run_routing(args) -> int:
     return routing.main(args)
 
 
+def run_reviewers(args) -> int:
+    from . import reviewers
+    return reviewers.main(args)
+
+
+def run_forecast(args) -> int:
+    from . import forecast
+    return forecast.main(args)
+
+
 def parser() -> argparse.ArgumentParser:
     top = argparse.ArgumentParser(prog='forge dataset', description='The run dataset: every run\'s inputs and outcomes.')
     sub = top.add_subparsers(dest='command', required=True)
@@ -204,6 +236,17 @@ def parser() -> argparse.ArgumentParser:
     p.add_argument('--since', type=float, metavar='DAYS', help='only tasks first attempted in the last DAYS days')
     p.add_argument('--json', action='store_true')
     p.set_defaults(func=run_routing)
+    p = sub.add_parser('reviewers', help='how often work each reviewer passed was later reverted or rewritten (advisory)')
+    p.add_argument('--horizon-days', type=int, default=14, help='the outcomes recorded at this horizon (default 14)')
+    p.add_argument('--repo', help='only this repo')
+    p.add_argument('--json', action='store_true')
+    p.set_defaults(func=run_reviewers)
+    p = sub.add_parser('forecast', help="a plan's likely tokens, time and first-try pass rate, from past tasks (advisory)")
+    p.add_argument('plan', help='the plan dir')
+    p.add_argument('--repo', help='only history from this repo')
+    p.add_argument('--json', action='store_true')
+    p.add_argument('--run', action='store_true', help=argparse.SUPPRESS)   # forge-parallel.sh run: budgeted, silent, records
+    p.set_defaults(func=run_forecast)
     p = sub.add_parser('event', help=argparse.SUPPRESS)
     p.add_argument('name')
     p.add_argument('--dir')

package/scripts/forge_dataset/export.py

@@ -213,6 +213,10 @@ class Runs:
         for e in store.of('dispatch'):
             if e['task']:
                 self.dispatches[(e['run'], e['task'], e['d'].get('role'))].append(e)
+        # Store order is import order. A reader that takes [-1] means the dispatch
+        # the attempt ended on, so each list is chronological; a tie keeps store order.
+        for rows in self.dispatches.values():
+            rows.sort(key=lambda e: (e['attempt'] or 0, e['d'].get('wall_started') or e['ts']))
         self.accepted = {(e['run'], e['d'].get('task')) for e in store.of('invocation')
                          if e['d'].get('command') == 'accept' and e['d'].get('exit') == 0}
         self.interrupted = {(e['run'], e['task']) for e in store.of('event')
@@ -558,8 +562,23 @@ def _float(value):
         return None
 
 
+def session_continued(event) -> bool:
+    """Whether this dispatch was asked to continue a reviewer session.
+
+    `continued` and `failed` both record the request. A missing field, on a dispatch
+    captured before the experiment, is a fresh session.
+    """
+    return (event.get('d') or {}).get('session_continue') is not None
+
+
 def _qa_rows(runs: Runs):
-    """Every QA dispatch with a verdict, with the diff it was shown and its task text."""
+    """Every independent QA dispatch with a verdict, with the diff it was shown and its task text.
+
+    A dispatch that continued a reviewer session is not an independent judgment, including
+    one whose continue failed after it had already printed a verdict. A fresh fallback in
+    the same round has no `session_continue` and stays. There is no switch to put the
+    continued ones back, so a slice never mixes the two.
+    """
     store = runs.store
     effort = {}
     for (run, task), rows in runs.attempts.items():
@@ -572,6 +591,8 @@ def _qa_rows(runs: Runs):
         if role != 'qa':
             continue
         for e in rows:
+            if session_continued(e):
+                continue
             d = e['d']
             if d.get('verdict') not in ('PASS', 'FAIL'):
                 continue
@@ -604,6 +625,12 @@ def shadow_pairs(runs: Runs) -> dict:
     Usable means the shadow ran to its end on an untouched snapshot (`shadow_review` says
     done), both reviewers gave a verdict, and the shadow reviewed the commit that attempt
     committed. Anything else would count noise in the shadow as disagreement.
+
+    The pair uses the attempt's last verdict-bearing primary. The dispatch list is
+    chronological, so that entry is the one the attempt ended on. A QA-only re-run keeps
+    the attempt number, so the candidates stay unfiltered until that last one is chosen:
+    an earlier fresh verdict must not stand in for a later continued session. The pair is
+    then dropped when that chosen primary continued a session.
     """
     ended = {}
     for e in runs.store.of('event'):
@@ -620,8 +647,14 @@ def shadow_pairs(runs: Runs) -> dict:
             primary = [d for d in runs.dispatches.get((run, task, 'qa'), [])
                        if d['attempt'] == n and d['d'].get('verdict') in ('PASS', 'FAIL')]
             reviewed = (shadow['d'].get('shadow') or {}).get('reviewed')
-            if primary and reviewed and reviewed == runs.reviewed.get((run, task, n)):
-                out[(run, task, n)] = (primary[-1], shadow)
+            if not (primary and reviewed and reviewed == runs.reviewed.get((run, task, n))):
+                continue
+            # Chosen after the list is built, and that list is chronological: filtering
+            # continued reviews out first would let an earlier fresh verdict stand in
+            # for the one the attempt ended on.
+            if session_continued(primary[-1]):
+                continue
+            out[(run, task, n)] = (primary[-1], shadow)
     return out