@giamme/forge 0.9.0 → 0.11.0

package/scripts/forge-ui.py

@@ -5,6 +5,13 @@
 prints a link to the run; every later `open` reuses it. State lives in
 `${XDG_STATE_HOME:-~/.local/state}/forge/ui`: `server.json` (what we started), `server.log`
 and `lock` (serialises launchers). The server exits by itself after FORGE_UI_IDLE_S idle seconds.
+
+FORGE_UI_READONLY=1 starts (or insists on) a server that answers no write. The mode is recorded
+with the server, and a running server of the other mode is replaced like one running old code.
+
+The server runs without the shell's FORGE_* variables, but the CLIs its writes call need the forge
+configuration (control.CLI_ENV_NAMES): `open` writes those to `cli-env.json` (0600) for the server to
+load, records them in server.json, and refuses (exit 5) to reuse a server started with other values.
 """
 from __future__ import annotations
 
@@ -38,6 +45,7 @@ LOCK_WAIT_S = 10
 START_WAIT_S = 5
 STOP_WAIT_S = 2
 BROWSER_WAIT_S = 2  # a browser command that hangs must not hold `open` past its budget
+CLI_ENV_FILE = 'cli-env.json'
 
 
 def state_dir() -> Path:
@@ -123,13 +131,13 @@ def read_state(state):
     return data if isinstance(data, dict) else None
 
 
-def write_state(state, data):
-    handle, temp = tempfile.mkstemp(prefix='server.json.', dir=str(state))
+def write_state(state, data, name='server.json'):
+    handle, temp = tempfile.mkstemp(prefix=name + '.', dir=str(state))  # mkstemp makes it 0600
     try:
         with os.fdopen(handle, 'w') as out:
             json.dump(data, out, indent=1, sort_keys=True)
             out.write('\n')
-        os.replace(temp, state / 'server.json')
+        os.replace(temp, state / name)
     except BaseException:
         try:
             os.unlink(temp)
@@ -187,12 +195,25 @@ def terminate(pid) -> bool:
     return wait_dead(pid, STOP_WAIT_S)
 
 
-def _spawn(state, port, registry, idle):
+def read_only_wanted() -> bool:
+    return os.environ.get('FORGE_UI_READONLY', '').strip().lower() in ('1', 'true', 'yes', 'on')
+
+
+def is_read_only(answer) -> bool:
+    """Whether a live /api/health answer is a read-only server (a server too old to say is one)."""
+    return answer.get('controls') is not True
+
+
+def _spawn(state, port, registry, idle, read_only=False, cli_env=None):
     """Start the server; returns (process, port) or (None, error text)."""
     log = state / 'server.log'
     env = {k: v for k, v in os.environ.items() if not k.startswith('FORGE_')}
-    command = [sys.executable, '-I', str(Path(__file__).resolve()), '_serve', '--port', str(port),
-               '--quiet', '--registry-dir', registry, '--idle-exit', str(idle)]
+    # The forge configuration goes to the server's CLIs only, through a file: never into its own environment.
+    write_state(state, cli_env or {}, CLI_ENV_FILE)
+    command = [sys.executable, '-I', str(Path(__file__).resolve()), '_serve', '--cli-env', str(state / CLI_ENV_FILE),
+               '--port', str(port), '--quiet', '--registry-dir', registry, '--idle-exit', str(idle)]
+    if read_only:  # the server's environment has no FORGE_* variables, so the mode goes as a flag
+        command.append('--read-only')
     with os.fdopen(open_regular(log, os.O_WRONLY | os.O_CREAT | os.O_TRUNC), 'w') as out:
         proc = subprocess.Popen(command, cwd=str(state), stdin=subprocess.DEVNULL, stdout=out, stderr=out,
                                 start_new_session=True, close_fds=True, env=env)
@@ -211,11 +232,11 @@ def _spawn(state, port, registry, idle):
     return None, None
 
 
-def start_server(state, port, registry, idle):
+def start_server(state, port, registry, idle, read_only=False, cli_env=None):
     """Spawn a server, falling back once to a free port when the preferred one is taken."""
-    proc, result = _spawn(state, port, registry, idle)
+    proc, result = _spawn(state, port, registry, idle, read_only, cli_env)
     if proc is None and result == 2 and port != 0:
-        proc, result = _spawn(state, 0, registry, idle)
+        proc, result = _spawn(state, 0, registry, idle, read_only, cli_env)
     if proc is None:
         return None
     return {'pid': proc.pid, 'port': result}
@@ -258,6 +279,14 @@ def browse(url):
     return 0
 
 
+def config_change(recorded, current):
+    """How the running server's forge configuration differs from this shell's, as text; '' when it does not."""
+    recorded = recorded if isinstance(recorded, dict) else {}  # a record without one has none
+    names = sorted(name for name in set(recorded) | set(current) if recorded.get(name) != current.get(name))
+    return '; '.join(f'{name} is {current.get(name, "unset")} here, {recorded.get(name, "unset")} in the server'
+                     for name in names)
+
+
 def run_info(directory):
     """(repo, None) for a run dir, or (None, why not). Never raises and never blocks."""
     marker = directory / 'repo'
@@ -269,7 +298,10 @@ def run_info(directory):
         # a FIFO or a device named `repo` would block the read: only a regular file is read
         if not stat.S_ISREG(os.stat(marker).st_mode):
             return None, f'{marker} is not a regular file'
-        repo = marker.read_text().strip()
+        # only the first line names the repo, as in forge_registry._repo_of (the runners register
+        # the same run dir under that line; anything after it must not change where it lands)
+        lines = marker.read_text().splitlines()
+        repo = lines[0].strip() if lines else ''
     except (OSError, ValueError) as exc:  # ValueError: not decodable text
         return None, f'cannot read {marker}: {getattr(exc, "strerror", None) or exc}'
     if not repo:
@@ -282,6 +314,7 @@ def run_info(directory):
 def cmd_open(args):
     import forge_registry
     from forge_ui import __version__, build_id
+    from forge_ui.control import cli_env_from
     from forge_ui.reader import run_key
 
     try:
@@ -307,6 +340,8 @@ def cmd_open(args):
     state = state_dir()
     log = state / 'server.log'
     registry = str(forge_registry.registry_dir())
+    read_only = read_only_wanted()
+    cli_env = cli_env_from(os.environ)
     spawned = None
     # Any OSError below is state we cannot use (state dir, lock, server.log, server.json) or a
     # spawn that failed: exit 4. An unreadable or invalid server.json is not one: it means no server.
@@ -317,26 +352,35 @@ def cmd_open(args):
             server = read_state(state)
             answer = confirmed(server) if server else None
             if answer:
-                # the record and the live answer must both name this code
-                if server.get('build') == build and answer.get('build') == build and server.get('registry') == registry:
-                    pass  # reuse
+                # the record and the live answer must both name this code and the mode asked for
+                if (server.get('build') == build and answer.get('build') == build and server.get('registry') == registry
+                        and bool(server.get('read_only')) == read_only and is_read_only(answer) == read_only):
+                    # Reused, it would run this shell's writes with another registry and dataset, and a
+                    # restart could cut off a write in flight: say so instead, and how to restart it.
+                    change = config_change(server.get('cli_env'), cli_env)
+                    if change:
+                        _say(f'error: the running forge-ui (pid {server["pid"]}) has another forge configuration '
+                             f'for its writes: {change}. Run `forge ui stop`, then open again.')
+                        return 5
                 else:
                     terminate(server['pid'])
                     server = None
             else:
                 server = None  # dead, or a pid that is not our forge-ui: never signal it
             if server is None:
-                spawned = start_server(state, port, registry, idle)
+                spawned = start_server(state, port, registry, idle, read_only, cli_env)
                 if spawned is None:
                     _say(f'error: the server did not start; see {log}')
                     return 4
-                server = dict(spawned, version=__version__, build=build, registry=registry, log=str(log),
-                              started=datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), opened=[])
+                server = dict(spawned, version=__version__, build=build, registry=registry, log=str(log), read_only=read_only,
+                              cli_env=cli_env, started=datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), opened=[])
             opened = server.get('opened')
             opened = opened if isinstance(opened, list) else []
             mark = key or ''
             # a key counts as opened only when a browser is really launched for it
-            first = mark not in opened and browser_wanted(args)
+            wanted = browser_wanted(args)
+            first = mark not in opened and wanted
+            again = mark in opened and wanted  # say so, or a repeated `open` looks like it did nothing
             if first:
                 opened.append(mark)
             server['opened'] = opened
@@ -354,6 +398,8 @@ def cmd_open(args):
     print(f'forge-ui: {url}')
     if first:
         launch_browser(url)
+    elif again:
+        _say('a browser tab was already opened for this link, so no new one was opened; use the link above')
     return 0
 
 
@@ -368,11 +414,13 @@ def cmd_status(args):
         return 1
     info = {'running': True, 'pid': server['pid'], 'port': server['port'],
             'url': f'http://{HOST}:{server["port"]}/', 'version': answer.get('version'),
-            'build': answer.get('build'), 'started': server.get('started'), 'log': server.get('log')}
+            'build': answer.get('build'), 'started': server.get('started'), 'log': server.get('log'),
+            'read_only': is_read_only(answer)}
     if args.json:
         print(json.dumps(info, sort_keys=True))
     else:
-        print(f'forge-ui: running at {info["url"]} (pid {info["pid"]}, started {info["started"]})')
+        mode = 'read-only' if info['read_only'] else 'read-write'
+        print(f'forge-ui: running at {info["url"]} (pid {info["pid"]}, started {info["started"]}, {mode})')
     return 0
 
 
@@ -403,8 +451,26 @@ def cmd_stop(args):
     return 0
 
 
+def load_cli_env(path):
+    """The forge configuration `_spawn` wrote for the server's CLIs; OSError or ValueError otherwise."""
+    from forge_ui import control
+    with os.fdopen(open_regular(path, os.O_RDONLY), 'rb') as handle:
+        return control.check_cli_env(json.loads(handle.read()))
+
+
 def serve(argv):
+    from forge_ui import control
     from forge_ui.__main__ import main
+    if argv[:1] == ['--cli-env']:
+        if len(argv) < 2:
+            _say('error: --cli-env needs a file')
+            return 1  # not 2: start_server reads 2 as a taken port
+        try:
+            control.cli_env = load_cli_env(argv[1])
+        except (OSError, ValueError, RecursionError) as exc:
+            _say(f'error: cannot load the CLI configuration {argv[1]}: {getattr(exc, "strerror", None) or exc}')
+            return 1
+        argv = argv[2:]
     return main(argv)
 
 

package/scripts/forge_dataset/__init__.py

@@ -18,6 +18,8 @@ can still be answered from old runs.
 Capture is fail-open in the same way Jev is. Every entry point swallows its own errors
 into errors.log, never prints, never raises, and bounds its wait on the database with the
 SQLite busy timeout. It uses no signals, because Fractal captures from worker threads.
+record_strict() is the one exception, for a caller that must know an event is stored (it
+raises DatasetError instead of returning 0).
 FORGE_DATASET=off turns it off.
 
 Nothing that builds a prompt may ever read this store: a reviewer that could see earlier
@@ -25,12 +27,14 @@ runs' verdicts would no longer be independent.
 """
 from __future__ import annotations
 
+import contextlib
 import gzip
 import hashlib
 import json
 import os
 import sqlite3
 import subprocess
+import threading
 import time
 import traceback
 import uuid
@@ -90,8 +94,13 @@ def data_home() -> Path:
     explicit = os.environ.get('FORGE_DATA_HOME')
     if explicit:
         return Path(explicit).expanduser()
-    base = os.environ.get('XDG_DATA_HOME') or str(Path.home() / '.local' / 'share')
-    return Path(base).expanduser() / 'forge'
+    # An empty or relative XDG_DATA_HOME counts as unset, as the XDG spec says; taken
+    # literally it named a directory below the current one, usually the user's repo.
+    value = os.environ.get('XDG_DATA_HOME') or ''
+    base = Path(value).expanduser() if value else None
+    if base is None or not base.is_absolute():
+        base = Path.home() / '.local' / 'share'
+    return base / 'forge'
 
 
 def dataset_dir() -> Path:
@@ -253,14 +262,46 @@ def put_blob(data: bytes | str, kind: str = 'text') -> dict:
                 n=len(data), kept=len(stored), cut=span)
 
 
+_BLOB_WATCH = threading.local()
+
+
+@contextlib.contextmanager
+def watch_blob_failures():
+    """Collect this thread's failed blob writes while the block runs (yields the list).
+
+    Only a caller that asks pays for the record: a long-lived process that never watches
+    keeps nothing, and another thread's failed write never lands in this thread's list.
+    """
+    previous = getattr(_BLOB_WATCH, 'failures', None)
+    failures: list[str] = []
+    _BLOB_WATCH.failures = failures
+    try:
+        yield failures
+    finally:
+        _BLOB_WATCH.failures = previous
+
+
 def put_file(path: str | Path, kind: str = 'text') -> dict | None:
+    """Store a file's content; None when it is absent, unreadable, or could not be stored.
+
+    A failed write into the blob store is also noted for a watch_blob_failures() block: the caller
+    still gets None (live capture fails open), but a strict importer can tell "no file" from
+    "content lost" and refuse to let the event win its rid without it.
+    """
     path = Path(path)
     try:
         if not path.is_file():
             return None
-        return put_blob(path.read_bytes(), kind)
+        data = path.read_bytes()
     except OSError:
         return None
+    try:
+        return put_blob(data, kind)
+    except OSError as error:
+        failures = getattr(_BLOB_WATCH, 'failures', None)
+        if failures is not None:
+            failures.append('%s: cannot store content: %s' % (path, error))
+        return None
 
 
 def read_blob(stored_sha: str) -> bytes:
@@ -384,13 +425,29 @@ def insert(db: sqlite3.Connection, event: dict) -> bool:
     return cursor.rowcount == 1
 
 
-def record(events: list[dict], repo: str | Path | None = None, where: str = 'record') -> int:
-    """Write events in one transaction. Fail-open: returns how many were new, 0 on error."""
-    if not enabled() or not events:
+class DatasetError(Exception):
+    """An event could not be persisted (or an import/archive could not be completed)."""
+
+
+def _record(events: list[dict], repo: str | Path | None, where: str, strict: bool) -> int:
+    """The one writer behind record() and record_strict(), so the two cannot drift.
+
+    Lenient: any problem is logged (or, for a refused location, ignored) and counts as 0.
+    Strict: the same problems raise DatasetError, because 0 new rows must keep meaning
+    "everything was already there".
+    """
+    if not events:
         return 0
     try:
-        # Refused silently: errors.log lives in the same refused place.
-        if location_refused(repo):
+        if not enabled():
+            if strict:
+                raise DatasetError('the dataset is off (FORGE_DATASET=off)')
+            return 0
+        refused = location_refused(repo)
+        if refused:
+            # Not logged: errors.log lives in the same refused place.
+            if strict:
+                raise DatasetError(refused)
             return 0
         # Fingerprinting the code reads every shipped file and runs git once per process:
         # done under the write lock, it held every other writer past its busy timeout.
@@ -414,11 +471,30 @@ def record(events: list[dict], repo: str | Path | None = None, where: str = 'rec
             raise
         finally:
             db.close()
+    except DatasetError:
+        raise
     except Exception as error:
         log_error(where, error)
+        if strict:
+            raise DatasetError('%s: %s' % (where, error)) from error
         return 0
 
 
+def record(events: list[dict], repo: str | Path | None = None, where: str = 'record') -> int:
+    """Write events in one transaction. Fail-open: returns how many were new, 0 on error."""
+    return _record(events, repo, where, strict=False)
+
+
+def record_strict(events: list[dict], repo: str | Path | None = None, where: str = 'record') -> int:
+    """Like record(), but a failure to persist raises DatasetError instead of returning 0.
+
+    For callers that must know every event is in the store (`forge runs rm` before it
+    deletes a run dir). Covers a refused location, a store that is off or unwritable, a
+    sqlite error and a database still locked after the busy timeout.
+    """
+    return _record(events, repo, where, strict=True)
+
+
 def create_once(path: Path, text: str) -> str:
     """Write `text` to `path` unless it exists; return whichever content won.
 

package/scripts/forge_dataset/capture.py

@@ -155,8 +155,11 @@ def point(name: str, directory: str | None, task: str | None = None, values: dic
                 refs[label] = put_blob(value, 'log')
         task = task or ctx.get('task')
         data['blobs'] = refs
-        rid = 'event:%s:%s' % (name, h.sha256(json.dumps([uid, task, time.time_ns(), data], sort_keys=True,
-                                                         default=str).encode())[:24])
+        if name in h.INTERVENTIONS:
+            rid = h.intervention_rid(name, uid, task, values or {})
+        else:
+            rid = 'event:%s:%s' % (name, h.sha256(json.dumps([uid, task, time.time_ns(), data], sort_keys=True,
+                                                             default=str).encode())[:24])
         record([dict(rid=rid, kind=kind, ts=time.time(), run=uid, run_id=ctx.get('run_id'),
                      mode=ctx.get('mode'), task=task, parent=env['parent'], inv=env['inv'], data=data)],
                repo=repo or ctx.get('repo'), where='event:' + name)

package/scripts/forge_dataset/cli.py

@@ -1,6 +1,6 @@
 """`forge dataset ...`: inspect the run dataset, import old runs, and the shell's capture hooks.
 
-User-facing: status, dump, import, export, outcomes. Internal (called by forge's shell scripts, fail-open,
+User-facing: status, dump, import, export, outcomes, routing, reviewers, forecast. Internal (called by forge's shell scripts, fail-open,
 silent): event, invocation, plan.
 """
 from __future__ import annotations
@@ -28,6 +28,24 @@ def _human(n: int) -> str:
     return str(n)
 
 
+def _continued_sessions(db) -> dict:
+    """How many dispatches asked to continue a reviewer session.
+
+    Counted from the dispatch events themselves. A shadow review is not required, and a
+    continue that failed still counts: the file was written because continuing was requested.
+    """
+    continued = failed = 0
+    for (raw,) in db.execute(
+            "SELECT data FROM events WHERE kind = 'dispatch' AND data LIKE ?",
+            ('%"session_continue": "%',)):
+        value = json.loads(raw).get('session_continue')
+        if value == 'continued':
+            continued += 1
+        elif value == 'failed':
+            failed += 1
+    return dict(continued_reviews=continued, failed_continues=failed)
+
+
 def status(args) -> int:
     from . import checks
     directory = dataset_dir()
@@ -43,6 +61,7 @@ def status(args) -> int:
             last = db.execute('SELECT MAX(ts) FROM events').fetchone()[0]
             summary['last_event'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last)) if last else None
             summary['checks'] = checks.run(db, verbose=args.verbose)
+            summary.update(_continued_sessions(db))
             if summary['events'].get('dispatch'):
                 from . import export
                 if db.execute("SELECT 1 FROM events WHERE kind = 'dispatch' AND data LIKE '%qa_shadow%' LIMIT 1").fetchone():
@@ -77,6 +96,9 @@ def status(args) -> int:
                   % (shadow['shadow_reviews'], shadow['pairs'], shadow['agree'], rate))
             for pair, n in shadow['by_pair'].items():
                 print('          %s: %d of %d agree' % (pair, n['agree'], n['pairs']))
+        if summary.get('continued_reviews') or summary.get('failed_continues'):
+            print('  continued reviews: %d' % summary['continued_reviews'])
+            print('  failed continues: %d' % summary['failed_continues'])
     if summary['legacy_archives']:
         print('  legacy: %d archived run dir(s) kept in %s (`forge dataset import %s` re-reads them;'
               ' events already stored are skipped)' % (summary['legacy_archives'], legacy, legacy))
@@ -163,6 +185,16 @@ def run_routing(args) -> int:
     return routing.main(args)
 
 
+def run_reviewers(args) -> int:
+    from . import reviewers
+    return reviewers.main(args)
+
+
+def run_forecast(args) -> int:
+    from . import forecast
+    return forecast.main(args)
+
+
 def parser() -> argparse.ArgumentParser:
     top = argparse.ArgumentParser(prog='forge dataset', description='The run dataset: every run\'s inputs and outcomes.')
     sub = top.add_subparsers(dest='command', required=True)
@@ -204,6 +236,17 @@ def parser() -> argparse.ArgumentParser:
     p.add_argument('--since', type=float, metavar='DAYS', help='only tasks first attempted in the last DAYS days')
     p.add_argument('--json', action='store_true')
     p.set_defaults(func=run_routing)
+    p = sub.add_parser('reviewers', help='how often work each reviewer passed was later reverted or rewritten (advisory)')
+    p.add_argument('--horizon-days', type=int, default=14, help='the outcomes recorded at this horizon (default 14)')
+    p.add_argument('--repo', help='only this repo')
+    p.add_argument('--json', action='store_true')
+    p.set_defaults(func=run_reviewers)
+    p = sub.add_parser('forecast', help="a plan's likely tokens, time and first-try pass rate, from past tasks (advisory)")
+    p.add_argument('plan', help='the plan dir')
+    p.add_argument('--repo', help='only history from this repo')
+    p.add_argument('--json', action='store_true')
+    p.add_argument('--run', action='store_true', help=argparse.SUPPRESS)   # forge-parallel.sh run: budgeted, silent, records
+    p.set_defaults(func=run_forecast)
     p = sub.add_parser('event', help=argparse.SUPPRESS)
     p.add_argument('name')
     p.add_argument('--dir')

package/scripts/forge_dataset/export.py

@@ -213,6 +213,10 @@ class Runs:
         for e in store.of('dispatch'):
             if e['task']:
                 self.dispatches[(e['run'], e['task'], e['d'].get('role'))].append(e)
+        # Store order is import order. A reader that takes [-1] means the dispatch
+        # the attempt ended on, so each list is chronological; a tie keeps store order.
+        for rows in self.dispatches.values():
+            rows.sort(key=lambda e: (e['attempt'] or 0, e['d'].get('wall_started') or e['ts']))
         self.accepted = {(e['run'], e['d'].get('task')) for e in store.of('invocation')
                          if e['d'].get('command') == 'accept' and e['d'].get('exit') == 0}
         self.interrupted = {(e['run'], e['task']) for e in store.of('event')
@@ -558,8 +562,23 @@ def _float(value):
         return None
 
 
+def session_continued(event) -> bool:
+    """Whether this dispatch was asked to continue a reviewer session.
+
+    `continued` and `failed` both record the request. A missing field, on a dispatch
+    captured before the experiment, is a fresh session.
+    """
+    return (event.get('d') or {}).get('session_continue') is not None
+
+
 def _qa_rows(runs: Runs):
-    """Every QA dispatch with a verdict, with the diff it was shown and its task text."""
+    """Every independent QA dispatch with a verdict, with the diff it was shown and its task text.
+
+    A dispatch that continued a reviewer session is not an independent judgment, including
+    one whose continue failed after it had already printed a verdict. A fresh fallback in
+    the same round has no `session_continue` and stays. There is no switch to put the
+    continued ones back, so a slice never mixes the two.
+    """
     store = runs.store
     effort = {}
     for (run, task), rows in runs.attempts.items():
@@ -572,6 +591,8 @@ def _qa_rows(runs: Runs):
         if role != 'qa':
             continue
         for e in rows:
+            if session_continued(e):
+                continue
             d = e['d']
             if d.get('verdict') not in ('PASS', 'FAIL'):
                 continue
@@ -604,6 +625,12 @@ def shadow_pairs(runs: Runs) -> dict:
     Usable means the shadow ran to its end on an untouched snapshot (`shadow_review` says
     done), both reviewers gave a verdict, and the shadow reviewed the commit that attempt
     committed. Anything else would count noise in the shadow as disagreement.
+
+    The pair uses the attempt's last verdict-bearing primary. The dispatch list is
+    chronological, so that entry is the one the attempt ended on. A QA-only re-run keeps
+    the attempt number, so the candidates stay unfiltered until that last one is chosen:
+    an earlier fresh verdict must not stand in for a later continued session. The pair is
+    then dropped when that chosen primary continued a session.
     """
     ended = {}
     for e in runs.store.of('event'):
@@ -620,8 +647,14 @@ def shadow_pairs(runs: Runs) -> dict:
             primary = [d for d in runs.dispatches.get((run, task, 'qa'), [])
                        if d['attempt'] == n and d['d'].get('verdict') in ('PASS', 'FAIL')]
             reviewed = (shadow['d'].get('shadow') or {}).get('reviewed')
-            if primary and reviewed and reviewed == runs.reviewed.get((run, task, n)):
-                out[(run, task, n)] = (primary[-1], shadow)
+            if not (primary and reviewed and reviewed == runs.reviewed.get((run, task, n))):
+                continue
+            # Chosen after the list is built, and that list is chronological: filtering
+            # continued reviews out first would let an earlier fresh verdict stand in
+            # for the one the attempt ended on.
+            if session_continued(primary[-1]):
+                continue
+            out[(run, task, n)] = (primary[-1], shadow)
     return out