@giamme/forge 0.0.0-stage → 0.8.0

package/scripts/forge_dataset/outcomes.py

@@ -0,0 +1,275 @@
+"""`forge dataset outcomes`: what happened to merged work after it landed, read from git.
+
+Every other outcome in the store is a reviewer's opinion. This one is not: once
+`integrate` fast-forwards a branch, the lines each task added either stay in the code or
+get rewritten or reverted. That is observable later, from the repo's history alone.
+
+`integrate` merges the run branch with --no-ff and fast-forwards the user's branch to that
+merge, never squashing, so every attempt commit lands as-is and `git blame` still
+attributes a task's surviving lines to its own commits (base_ref..reviewed_commit).
+For each `integrate_advanced` event this records, per task it brought in, how many of the
+lines the task added are still attributed to the task's commits at a horizon (14 days by
+default) after the integration, and whether any later commit reverted it.
+
+Survival describes the artifact that landed (with every retry it took), not the first
+model that was routed to the task: the exporter attaches it to the review of that diff.
+
+Read-only by construction: the store is opened read-only (connect() would migrate it),
+and git runs only object-store queries (rev-parse, rev-list, log, diff between commits,
+blame at a commit, cat-file) with GIT_OPTIONAL_LOCKS=0, so nothing touches a working
+tree or an index. Results are written with record(), which refuses a store inside the
+repo. A horizon still in the future, or a query that failed, writes nothing and is tried
+again next time; the rid names the integration, the reviewed commit and the horizon, so
+a repeat harvest adds nothing.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sqlite3
+import subprocess
+import time
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from . import dataset_dir, record, sha256
+
+HORIZON_DAYS = 14
+GIT_TIMEOUT_S = 120
+_REVERTS = re.compile(r'This reverts commit ([0-9a-f]{7,40})')
+_HUNK = re.compile(rb'^([0-9a-f]{40}) \d+ \d+ (\d+)$')
+
+
+class GitError(Exception):
+    pass
+
+
+class Git:
+    """Object-store queries in one repo. A failure raises; it never reads as "nothing found"."""
+
+    def __init__(self, repo: str):
+        self.repo = repo
+        self.env = dict(os.environ, GIT_OPTIONAL_LOCKS='0', GIT_TERMINAL_PROMPT='0', LC_ALL='C')
+
+    def out(self, *args, ok=(0,)) -> tuple[int, bytes]:
+        try:
+            result = subprocess.run(['git', '-C', self.repo, *args], stdout=subprocess.PIPE,
+                                    stderr=subprocess.DEVNULL, timeout=GIT_TIMEOUT_S, env=self.env)
+        except (OSError, subprocess.SubprocessError) as error:
+            raise GitError('%s: %s' % (args[0], error)) from error
+        if result.returncode not in ok:
+            raise GitError('%s exited %d' % (args[0], result.returncode))
+        return result.returncode, result.stdout
+
+    def text(self, *args) -> str:
+        return self.out(*args)[1].decode('utf-8', 'replace')
+
+    def exists(self, obj: str) -> bool:
+        return self.out('cat-file', '-e', obj, ok=(0, 1, 128))[0] == 0
+
+    def blamed(self, rev: str, path: str, commits: set) -> int:
+        """Lines of `path` at `rev` whose origin is one of `commits`, streamed, never capped."""
+        try:
+            proc = subprocess.Popen(['git', '-C', self.repo, 'blame', '--incremental', rev, '--', path],
+                                    stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=self.env)
+        except OSError as error:
+            raise GitError('blame: %s' % error) from error
+        count = 0
+        try:
+            for line in proc.stdout:
+                match = _HUNK.match(line.rstrip(b'\n'))
+                if match and match.group(1).decode() in commits:
+                    count += int(match.group(2))
+            if proc.wait(timeout=GIT_TIMEOUT_S):
+                raise GitError('blame %s exited %d' % (path, proc.returncode))
+        except subprocess.TimeoutExpired as error:
+            proc.kill()
+            raise GitError('blame %s timed out' % path) from error
+        finally:
+            proc.stdout.close()
+        return count
+
+
+def read_store():
+    """Integrations, merges and attempts, from a read-only connection; None without a store."""
+    path = dataset_dir() / 'dataset.sqlite'
+    if not path.exists():
+        return None
+    db = sqlite3.connect('file:%s?mode=ro' % path, uri=True, timeout=5)
+    try:
+        rows = db.execute("SELECT kind, run, task, repo, ts, data FROM events WHERE kind IN ('event', 'attempt')"
+                          ).fetchall()
+        repos = dict(db.execute('SELECT repo, path FROM repos'))
+        have = {rid for (rid,) in db.execute("SELECT rid FROM events WHERE kind = 'outcome'")}
+    finally:
+        db.close()
+    integrations, merges, bases = [], {}, {}
+    for kind, run, task, repo, ts, data in rows:
+        d = json.loads(data)
+        if kind == 'attempt':
+            if d.get('reviewed_commit') and d.get('base_ref'):
+                bases[d['reviewed_commit']] = d['base_ref']
+        elif d.get('name') == 'integrate_advanced' and d.get('candidate') and d.get('parent'):
+            integrations.append(dict(run=run, repo=repo, ts=ts, **{k: d.get(k) for k in
+                                                                   ('branch', 'parent', 'candidate', 'target')}))
+        elif d.get('name') == 'merge' and d.get('status') == 'MERGED' and task:
+            commit = d.get('commit') or d.get('reviewed_commit')
+            if commit:
+                # The last MERGED event for a commit wins; split/combine copies keep their run.
+                merges[(repo, commit)] = dict(run=run, task=task, commit=commit, integration=d.get('integration'))
+    return dict(integrations=sorted(integrations, key=lambda i: i['ts']), merges=merges, bases=bases,
+                repos=repos, have=have)
+
+
+def outcome_rid(candidate: str, reviewed: str, horizon_days: int) -> str:
+    return 'outcome:survival:' + sha256(('%s:%s:%d' % (candidate, reviewed, horizon_days)).encode())[:24]
+
+
+def horizon_rev(git: Git, branch: str, candidate: str, cutoff: float, now: float):
+    """('ok', rev, ts) | ('lost', None, None) | ('pending', None, None).
+
+    The branch is resolved once to an immutable tip. The candidate must sit on its
+    first-parent chain (a rebase, squash or reset loses it: unobservable, not failure).
+    The horizon is found by position along that chain, so out-of-order commit dates
+    cannot jump past a commit that came later.
+    """
+    if not branch or git.out('rev-parse', '--verify', '-q', branch + '^{commit}', ok=(0, 1))[0]:
+        return 'lost', None, None
+    tip = git.text('rev-parse', branch + '^{commit}').strip()
+    chain = []                            # first-parent commits after the candidate, newest first
+    for line in git.text('log', '--first-parent', '--format=%H %ct %P', '%s..%s' % (candidate, tip)).splitlines():
+        sha, ct, *parents = line.split()
+        chain.append((sha, int(ct), parents))
+    if tip != candidate and (not chain or chain[-1][2][:1] != [candidate]):
+        return 'lost', None, None         # the walk never passed through the candidate
+    chain = [(sha, ct) for sha, ct, _ in reversed(chain)]
+    candidate_ct = int(git.text('log', '-1', '--format=%ct', candidate).strip())
+    rev, rev_ts = candidate, candidate_ct
+    for sha, ct in chain:
+        if ct > cutoff:
+            return 'ok', rev, rev_ts
+        rev, rev_ts = sha, ct
+    if now < cutoff:
+        return 'pending', None, None      # more commits may still land before the horizon
+    return 'ok', rev, rev_ts
+
+
+def survival(git: Git, base: str, reviewed: str, rev: str) -> dict:
+    commits = set(git.text('rev-list', '%s..%s' % (base, reviewed)).split())
+    files, added_total, surviving_total = {}, 0, 0
+    for entry in git.text('diff', '--numstat', '-z', '--no-renames', base, reviewed).split('\0'):
+        cells = entry.lstrip('\n').split('\t', 2)
+        if len(cells) != 3 or cells[0] == '-':
+            continue                       # binary: no lines to follow
+        added, path = int(cells[0]), cells[2]
+        if added == 0:
+            continue
+        kept = git.blamed(rev, path, commits) if git.exists('%s:%s' % (rev, path)) else 0
+        kept = min(kept, added)
+        files[path] = [added, kept]
+        added_total += added
+        surviving_total += kept
+    return dict(commits=sorted(commits), files=files, added=added_total, surviving=surviving_total)
+
+
+def reverts(git: Git, candidate: str, rev: str, shas: set) -> list:
+    if rev == candidate:
+        return []
+    out = []
+    for block in git.text('log', '--format=%H%x00%B%x01', '%s..%s' % (candidate, rev)).split('\x01'):
+        sha, _, body = block.strip('\n').partition('\x00')
+        if any(any(s.startswith(m) for s in shas) for m in _REVERTS.findall(body)):
+            out.append(sha)
+    return out
+
+
+def harvest(*, horizon_days: int = HORIZON_DAYS, repo: str | None = None, include_temp: bool = False,
+            now: float | None = None) -> tuple[list, Counter]:
+    from .importer import is_temp
+    state = read_store()
+    counts = Counter()
+    if state is None:
+        return [], counts
+    now = time.time() if now is None else now
+    wanted = str(Path(repo).resolve()) if repo else None
+    by_repo = defaultdict(list)
+    for integration in state['integrations']:
+        path = state['repos'].get(integration['repo'])
+        if not path or (wanted and str(Path(path).resolve()) != wanted):
+            continue
+        if is_temp(path) and not include_temp:
+            counts['skipped_temp'] += 1
+            continue
+        if not Path(path).is_dir():
+            counts['repo_missing'] += 1
+            continue
+        by_repo[path].append(integration)
+    out = []
+    for path, integrations in by_repo.items():
+        git = Git(path)
+        for integration in integrations:
+            candidate, parent = integration['candidate'], integration['parent']
+            try:
+                if not git.exists(candidate + '^{commit}'):
+                    counts['candidate_missing'] += 1
+                    continue
+                landed = set(git.text('rev-list', candidate, '^' + parent).split())
+                tasks = [m for (r, c), m in state['merges'].items() if r == integration['repo'] and c in landed]
+                todo = [m for m in tasks if outcome_rid(candidate, m['commit'], horizon_days) not in state['have']]
+                counts['already'] += len(tasks) - len(todo)
+                if not todo:
+                    continue
+                cutoff = integration['ts'] + horizon_days * 86400
+                status, rev, rev_ts = horizon_rev(git, integration['branch'], candidate, cutoff, now)
+            except GitError:
+                counts['git_error'] += 1
+                continue
+            if status == 'pending':
+                counts['pending'] += len(todo)
+                continue
+            for merge in todo:
+                base = state['bases'].get(merge['commit'])
+                if not base:
+                    counts['no_base'] += 1
+                    continue
+                data = dict(name='survival', candidate=candidate, reviewed=merge['commit'], base_ref=base,
+                            branch=integration['branch'], horizon_days=horizon_days, integrated_ts=integration['ts'],
+                            horizon_rev=rev, horizon_ts=rev_ts, integrate_run=integration['run'])
+                try:
+                    if status == 'ok':
+                        found = survival(git, base, merge['commit'], rev)
+                        task_shas = set(found.pop('commits')) | {s for s in (merge.get('integration'), candidate) if s}
+                        data.update(found, reverted_by=reverts(git, candidate, rev, task_shas),
+                                    status='ok' if found['added'] else 'no_additions')
+                    else:
+                        data.update(status='lost')
+                except GitError:
+                    counts['git_error'] += 1
+                    continue
+                counts[data['status']] += 1
+                out.append((path, dict(rid=outcome_rid(candidate, merge['commit'], horizon_days), kind='outcome',
+                                       src='harvest', ts=now, run=merge['run'], task=merge['task'],
+                                       mode='parallel', data=data)))
+    return out, counts
+
+
+def main(args) -> int:
+    events, counts = harvest(horizon_days=args.horizon_days, repo=args.repo, include_temp=args.include_temp)
+    written = 0
+    if not args.dry_run:
+        by_repo = defaultdict(list)
+        for path, event in events:
+            by_repo[path].append(event)
+        for path, batch in by_repo.items():
+            written += record(batch, repo=path, where='outcomes')
+    label = 'would record' if args.dry_run else 'recorded'
+    print('%s %d survival outcome(s) at %d days' % (label, len(events) if args.dry_run else written,
+                                                     args.horizon_days))
+    for key in ('ok', 'no_additions', 'lost', 'pending', 'already', 'no_base', 'candidate_missing',
+                'repo_missing', 'skipped_temp', 'git_error'):
+        if counts.get(key):
+            print('  %-17s %d' % (key, counts[key]))
+    if not args.dry_run and written < len(events):
+        print('  %d not written; see %s' % (len(events) - written, dataset_dir() / 'errors.log'))
+    return 0

package/scripts/forge_dataset/routing.py

@@ -0,0 +1,278 @@
+"""`forge dataset routing`: how each model does per tier, and what the randomized picks say.
+
+Two tables, kept apart because they answer different questions:
+
+- **Observational**: every task's first attempt, by declared tier and the model and effort
+  that ran it (resolved, so `sol` and `gpt-6.1-sol:medium:codex` are one row). First-pass rate
+  (Wilson 95%), the share that passed review at some attempt (or that a human accepted),
+  unfinished tasks, attempts per task and the implementer's runtime. Routing chose who ran what, so a model sent the hard tasks looks
+  worse here than it is. Fractal runs are left out: their dispatches have their own identities
+  and their runtime would be missing.
+- **Randomized**: the `plan --explore` draws (forge-route.py), one row per (tier, default,
+  alternative) pair, since round-robin pools make the pair differ between rows and a spec has
+  no chance at all outside its own pairs. Each task counts in the arm it was *assigned*
+  (intent to treat), whatever the approval gate or a retry did afterwards; tasks with no
+  outcome are attrition, reported per arm. First-pass per arm is a Hajek (self-normalized
+  inverse-propensity) mean, the difference has a run-cluster bootstrap interval, and Kish's
+  effective n and the number of runs say how much the interval can be trusted.
+
+The decision rule is fixed in advance and only advises: a pair is judged at the first of 30,
+60 or 120 alternative assignments it reaches, on exactly the assignments up to that point.
+The alternative is the stronger model for a low-tier pair (switch if the interval is above
+0) and the cheaper one for medium and high (non-inferior if the interval's lower end is above
+-0.10 and the alternative's runtime is lower). Nothing here changes a plan.
+"""
+from __future__ import annotations
+
+import json
+import math
+import os
+import random
+import statistics
+import time
+from collections import defaultdict
+
+from . import connect
+from .export import Runs, Store, _attempts
+
+WINDOWS = (30, 60, 120)
+NONINFERIORITY = -0.10
+BOOTSTRAP = 2000
+MIN_ALTERNATIVES, MIN_RUNS = 10, 3
+
+
+def wilson(k: int, n: int, z: float = 1.96):
+    if not n:
+        return None, None
+    p = k / n
+    centre = (p + z * z / (2 * n)) / (1 + z * z / n)
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+def decision_before(decisions, started):
+    """The routing decision attempt 1 ran under: the latest recorded before it started.
+
+    A re-plan after attempt 1 began (or one with a new draw) must not relabel what that
+    attempt was assigned. Decisions repeat across plan revisions, so they are deduplicated.
+    """
+    unique = {}
+    for d in decisions or ():
+        unique[d.get('decision') or json.dumps(d, sort_keys=True)] = d
+    ordered = sorted(unique.values(), key=lambda d: d.get('at') or 0)
+    if not ordered:
+        return None
+    if started is None:
+        return ordered[-1]
+    before = [d for d in ordered if (d.get('at') or 0) <= started + 1]
+    return before[-1] if before else None
+
+
+def _started(runs: Runs, run, task, first) -> float | None:
+    for e in runs.dispatches.get((run, task, 'dwarf'), []):
+        if e['attempt'] == first['n'] and e['d'].get('wall_started'):
+            return float(e['d']['wall_started'])
+    return first['events'][0]['ts'] if first else None
+
+
+def _runtime(runs: Runs, run, task) -> float | None:
+    durations = [e['d'].get('duration_s') for e in runs.dispatches.get((run, task, 'dwarf'), [])]
+    durations = [x for x in durations if isinstance(x, (int, float))]
+    return sum(durations) if durations else None
+
+
+def tasks(runs: Runs, *, repo=None, since=None) -> list:
+    """One record per (run, task) that a plan held, with its routing decision and outcome."""
+    out = []
+    for run, revisions in runs.plans.items():
+        if any((p['d'].get('fractal_selection') or {}).get('enabled') for p in revisions):
+            continue
+        rows, decisions = {}, defaultdict(list)
+        for plan in revisions:
+            for tid, row in (plan['d'].get('tasks') or {}).items():
+                rows[tid] = row
+                decisions[tid].extend(row.get('route') or [])
+        for tid, row in rows.items():
+            events = runs.attempts.get((run, tid), [])
+            attempts = _attempts(runs, run, tid, events)
+            first = attempts[0] if attempts else None
+            if repo and not (events and runs.store.repo_paths.get(events[0]['repo']) == repo):
+                continue
+            if since and not (events and events[0]['ts'] >= since):
+                continue
+            started = _started(runs, run, tid, first) if first else None
+            decision = decision_before(decisions.get(tid), started)
+            resolved = runs.resolved(run, tid, 'dwarf', first['n']) if first else {}
+            model = ':'.join(x for x in (resolved.get('model'), resolved.get('effort')) if x) or None
+            out.append(dict(
+                run=run, task=tid, tier=row.get('declared_difficulty') or row.get('difficulty'),
+                ran=first['events'][0]['d'].get('dwarf') if first else None, model=model,
+                first_pass=first['outcome'] if first else None,
+                censored=(first['reason'] if first and first['outcome'] is None else (None if first else 'not_run')),
+                passed=any(a['outcome'] for a in attempts) or (run, tid) in runs.accepted,
+                attempts=len(attempts), runtime_s=_runtime(runs, run, tid), decision=decision))
+    return out
+
+
+def observational(records) -> list:
+    cells = defaultdict(list)
+    for r in records:
+        if (r['model'] or r['ran']) and r['first_pass'] is not None:
+            cells[(r['tier'], r['model'] or r['ran'])].append(r)
+    rows = []
+    for (tier, spec), group in sorted(cells.items(), key=lambda kv: (str(kv[0][0]), str(kv[0][1]))):
+        k = sum(1 for r in group if r['first_pass'])
+        low, high = wilson(k, len(group))
+        runtimes = [r['runtime_s'] for r in group if r['runtime_s'] is not None]
+        rows.append(dict(tier=tier, spec=spec, tasks=len(group), first_pass=k / len(group), ci=[low, high],
+                         passed=sum(1 for r in group if r['passed']) / len(group),
+                         unfinished=sum(1 for r in group if not r['passed']),
+                         attempts=statistics.mean(r['attempts'] for r in group),
+                         runtime_min=statistics.median(runtimes) / 60 if runtimes else None))
+    return rows
+
+
+def _hajek(items):
+    """(weighted first-pass, Kish effective n) over items with an outcome."""
+    known = [(1 / i['propensity'], 1.0 if i['first_pass'] else 0.0) for i in items if i['first_pass'] is not None]
+    total = sum(w for w, _ in known)
+    if not total:
+        return None, 0.0
+    return sum(w * y for w, y in known) / total, total * total / sum(w * w for w, _ in known)
+
+
+def _difference(items):
+    default = [i for i in items if i['arm'] == 'default']
+    alternative = [i for i in items if i['arm'] == 'alternative']
+    a, _ = _hajek(alternative)
+    d, _ = _hajek(default)
+    return None if a is None or d is None else a - d
+
+
+def _bootstrap(items, rng):
+    by_run = defaultdict(list)
+    for i in items:
+        by_run[i['run']].append(i)
+    runs = sorted(by_run, key=str)
+    diffs = []
+    for _ in range(BOOTSTRAP):
+        sample = [x for run in (rng.choice(runs) for _ in runs) for x in by_run[run]]
+        diff = _difference(sample)
+        if diff is not None:
+            diffs.append(diff)
+    if len(diffs) < BOOTSTRAP // 2:
+        return None
+    diffs.sort()
+    return [diffs[int(0.025 * len(diffs))], diffs[int(0.975 * len(diffs)) - 1]]
+
+
+def randomized(records, seed: int = 7) -> list:
+    pairs = defaultdict(list)
+    old_draws = 0
+    for r in records:
+        d = r['decision']
+        candidates = (d or {}).get('candidates') or []
+        if not d:
+            continue
+        if len(candidates) != 2 or not d.get('propensity'):
+            old_draws += 1                # an old all-specs draw has no fixed pair to compare in
+            continue
+        alternative = next(c for c in candidates if c != d.get('default'))
+        pairs[(d.get('tier'), d.get('default'), alternative)].append(dict(
+            r, arm='alternative' if d.get('chosen') == alternative else 'default',
+            propensity=float(d['propensity']), at=d.get('at') or 0,
+            followed=r['ran'] == d.get('chosen') if r['ran'] else None))
+    rows = []
+    rng = random.Random(seed)
+    if old_draws:
+        rows.append(dict(note='%d older exploratory draw(s) picked among every spec of their plan; they have no '
+                              'fixed pair to compare in and are left out' % old_draws))
+    for (tier, default, alternative), items in sorted(pairs.items(), key=lambda kv: tuple(map(str, kv[0]))):
+        items.sort(key=lambda i: i['at'])
+        n_alt = sum(1 for i in items if i['arm'] == 'alternative')
+        window = max((w for w in WINDOWS if n_alt >= w), default=None)
+        judged = items
+        if window:
+            cut = [i for i in items if i['arm'] == 'alternative'][window - 1]['at']
+            judged = [i for i in items if i['at'] <= cut]
+        arms = {}
+        for arm in ('default', 'alternative'):
+            group = [i for i in judged if i['arm'] == arm]
+            rate, n_eff = _hajek(group)
+            runtimes = [i['runtime_s'] for i in group if i['runtime_s'] is not None]
+            arms[arm] = dict(spec=default if arm == 'default' else alternative, assigned=len(group),
+                             first_pass=rate, n_eff=round(n_eff, 1),
+                             not_followed=sum(1 for i in group if i['followed'] is False),
+                             no_outcome=sum(1 for i in group if i['first_pass'] is None),
+                             runtime_min=statistics.median(runtimes) / 60 if runtimes else None)
+        run_count = len({i['run'] for i in judged})
+        diff = _difference(judged)
+        ci = (_bootstrap(judged, rng) if arms['alternative']['assigned'] >= MIN_ALTERNATIVES
+              and run_count >= MIN_RUNS else None)
+        stronger_alternative = tier == 'low'
+        if not window:
+            verdict = 'not enough randomized data (%d/%d alternative assignments)' % (n_alt, WINDOWS[0])
+        elif ci is None:
+            verdict = 'window %d reached, but too few runs for an interval' % window
+        elif stronger_alternative:
+            verdict = ('switch to %s: it passes more (interval above 0)' % alternative if ci[0] > 0
+                       else 'keep %s: %s is not shown to pass more' % (default, alternative))
+        else:
+            cheaper_faster = (arms['alternative']['runtime_min'] or math.inf) < (arms['default']['runtime_min'] or 0)
+            verdict = ('%s is non-inferior and faster: worth making the default' % alternative
+                       if ci[0] > NONINFERIORITY and cheaper_faster
+                       else 'keep %s: %s is not shown to be non-inferior' % (default, alternative))
+        rows.append(dict(tier=tier, default=default, alternative=alternative, alternative_is='stronger'
+                         if stronger_alternative else 'cheaper', window=window, runs=run_count, arms=arms,
+                         difference=diff, ci=ci, verdict=verdict))
+    return rows
+
+
+def _pct(x):
+    return '   -' if x is None else '%3.0f%%' % (100 * x)
+
+
+def main(args) -> int:
+    db = connect()
+    try:
+        runs = Runs(Store(db))
+    finally:
+        db.close()
+    repo = os.path.realpath(args.repo) if args.repo else None
+    since = time.time() - args.since * 86400 if args.since else None
+    records = tasks(runs, repo=repo, since=since)
+    report = dict(observational=observational(records), randomized=randomized(records))
+    if args.json:
+        print(json.dumps(report, indent=2, sort_keys=True))
+        return 0
+    print('observational: first attempts by declared tier and the model that ran them')
+    print('  (routing chose who ran what, so a model given the hard tasks looks worse than it is)')
+    print('  %-7s %-26s %5s %6s %-13s %6s %10s %8s %8s' % ('tier', 'model:effort', 'tasks', 'first', '95%',
+                                                          'passed', 'unfinished', 'attempts', 'dwarf min'))
+    for r in report['observational']:
+        low, high = r['ci']
+        print('  %-7s %-26s %5d %6s [%s-%s] %6s %10d %8.1f %8s' % (
+            r['tier'], r['spec'], r['tasks'], _pct(r['first_pass']), _pct(low).strip(), _pct(high).strip(),
+            _pct(r['passed']), r['unfinished'], r['attempts'],
+            '-' if r['runtime_min'] is None else '%.1f' % r['runtime_min']))
+    print()
+    print('randomized (plan --explore): each pair of models, tasks counted in the arm they were assigned')
+    pairs = [r for r in report['randomized'] if 'note' not in r]
+    for r in report['randomized']:
+        if 'note' in r:
+            print('  (%s)' % r['note'])
+    if not pairs:
+        print('  no exploratory decisions in pairs yet; `plan` explores at 0.2 by default')
+    for r in pairs:
+        d, a = r['arms']['default'], r['arms']['alternative']
+        print('  %s: %s (default) vs %s (%s), %d run(s)%s' % (
+            r['tier'], r['default'], r['alternative'], r['alternative_is'], r['runs'],
+            ', judged at window %d' % r['window'] if r['window'] else ''))
+        for name, arm in (('default', d), ('alternative', a)):
+            print('    %-11s %-24s assigned %3d  first-pass %s  n_eff %5.1f  not followed %d  no outcome %d  dwarf min %s' % (
+                name, arm['spec'], arm['assigned'], _pct(arm['first_pass']), arm['n_eff'], arm['not_followed'],
+                arm['no_outcome'], '-' if arm['runtime_min'] is None else '%.1f' % arm['runtime_min']))
+        if r['difference'] is not None:
+            print('    difference %+.2f%s' % (r['difference'], '' if not r['ci'] else '  95%% [%+.2f, %+.2f]' % tuple(r['ci'])))
+        print('    -> %s' % r['verdict'])
+    return 0

package/scripts/forge_fractal/__init__.py

@@ -0,0 +1,83 @@
+"""Opt-in Fractal adapter. Importing this package never imports Fractal."""
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+
+SCRIPTS = Path(__file__).resolve().parents[1]
+REVISION = '18793200c0d7e8cdb2db369ea3abe5647a1e15e4'
+VERSION = '1.2.0'
+ARCHIVE_SHA256 = '3f7714ee0baec05af8d856db2b00d63398026fae2fcf50edbee542671126072c'
+DEFAULTS = dict(depth=2, children=3, nodes=12, iterations=6, concurrency=3, deadline=2700)
+
+
+def state_root() -> Path:
+    return Path(os.environ.get('XDG_STATE_HOME', str(Path.home() / '.local/state'))).expanduser().resolve() / 'forge/fractal'
+
+
+def runtime_python() -> Path:
+    return state_root() / 'runtime/bin/python'
+
+
+def write_json(path: Path, data: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    _runtime.write_json(path, data)
+
+
+_runtime_spec = importlib.util.spec_from_file_location('forge_runtime', SCRIPTS / 'forge-runtime.py')
+_runtime = importlib.util.module_from_spec(_runtime_spec)
+_runtime_spec.loader.exec_module(_runtime)
+
+
+def read_json(path: Path) -> dict:
+    return json.loads(path.read_text())
+
+
+def identifier(value: str) -> str:
+    if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_-]{0,100}', value):
+        raise ValueError('Invalid managed identifier: ' + repr(value))
+    return value
+
+
+def managed_run(value: str) -> Path:
+    root = state_root() / 'runs'
+    path = root / identifier(value)
+    if path.is_symlink() or path.resolve().parent != root.resolve():
+        raise ValueError('Run must be a direct managed directory')
+    if not path.is_dir() or (path / 'run.json').is_symlink():
+        raise ValueError('Unknown managed run: ' + value)
+    return path
+
+
+def safe_file(root: Path, relative: str) -> Path:
+    path = root / relative
+    if path.resolve() != path.absolute() or not path.resolve().is_relative_to(root.resolve()):
+        raise ValueError('Artifact path escapes managed data or contains a symlink')
+    return path
+
+
+@contextlib.contextmanager
+def lock(path: Path, blocking: bool = True):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('a') as handle:
+        fcntl.flock(handle, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
+        yield
+
+
+def command(argv: list, *, cwd: Path | None = None, timeout: float = 120, input: bytes | None = None) -> bytes:
+    result = subprocess.run(list(map(str, argv)), cwd=cwd, input=input,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
+    if result.returncode:
+        raise RuntimeError(result.stderr.decode(errors='replace') or result.stdout.decode(errors='replace'))
+    return result.stdout
+
+
+def artifact(verb: str, *args: object) -> str:
+    return command(['/bin/bash', '-c', 'source "$1"; shift; "$@"', 'forge',
+                    SCRIPTS / 'forge-artifact.sh', verb, *args]).decode().strip()