@giamme/forge 0.11.1 → 0.12.0

package/scripts/forge_dataset/forecast.py

@@ -243,13 +243,25 @@ def _pct(v):
     return '%.0f%%' % (v * 100)
 
 
-def lines(fc: dict) -> list:
+def _names(rows: list, most: int = 8) -> str:
+    names = [r['task'] for r in rows]
+    return ', '.join(names[:most]) + (', ...' if len(names) > most else '')
+
+
+def lines(fc: dict, brief: bool = False) -> list:
+    """The forecast as text. brief (the block `run` prints) gives tasks without enough history
+    one line together instead of one row each, and no table at all when no task has enough."""
+    thin = [r for r in fc['tasks'] if not r['enough']]
+    if brief and len(thin) == len(fc['tasks']):
+        return ['no forecast yet: none of the %d task(s) has %d past tasks at its tier and model '
+                '(forge dataset forecast %s shows each count)' % (len(thin), MIN_HISTORY, fc['plan'])]
     out = ['forecast from %d past task(s) at the same tier and model (advisory; routing never reads it)'
            % fc['history_tasks']]
     for r in fc['tasks']:
         where = '%s %s' % (r['tier'] or '?', r['spec'] or '?')
         if not r['enough']:
-            out.append('  %-14s %-22s %s' % (r['task'], where, r['note']))
+            if not brief:
+                out.append('  %-14s %-22s %s' % (r['task'], where, r['note']))
             continue
         fp = r['first_pass']
         t, s = r['tokens'], r['time_s']
@@ -257,6 +269,9 @@ def lines(fc: dict) -> list:
             r['task'], where, _pct(fp['rate']), _pct(fp['ci'][0]), _pct(fp['ci'][1]), fp['k'], fp['n'],
             _tokens(t['median']), _tokens(t['p90']), _time(s['median']), _time(s['p90']),
             '  <- likely to need a retry' if r['flag'] else ''))
+    if brief and thin:
+        out.append('  %d task(s) without enough history yet (under %d past tasks): %s'
+                   % (len(thin), MIN_HISTORY, _names(thin)))
     if fc['merged']:
         out.append('  %d task(s) already merged, not forecast' % len(fc['merged']))
     tot = fc['total']
@@ -275,12 +290,17 @@ def lines(fc: dict) -> list:
     return out
 
 
+def _tmp(plan: Path) -> Path:
+    """Where save() writes forecast.json before moving it into place."""
+    return plan / ('.forecast.json.%d' % os.getpid())
+
+
 def save(plan: Path, fc: dict) -> None:
     """Keep what was predicted, so it can be checked against what happened. Fail-open."""
     body = {k: v for k, v in fc.items() if k != 'at'}
     digest = hashlib.sha256(json.dumps(body, sort_keys=True).encode()).hexdigest()[:16]
+    tmp = _tmp(plan)
     try:
-        tmp = plan / ('.forecast.json.%d' % os.getpid())
         tmp.write_text(json.dumps(fc, indent=2, sort_keys=True) + '\n')
         tmp.replace(plan / 'forecast.json')
     except OSError:
@@ -302,7 +322,7 @@ def advise(plan: Path, shown: list) -> None:
         return
     if not fc or not fc['tasks']:
         return
-    for line in lines(fc):
+    for line in lines(fc, brief=True):
         print(line)
     sys.stdout.flush()
     shown.append(True)
@@ -336,6 +356,11 @@ def main(args) -> int:
             if not shown:
                 print('forecast skipped: reading the history took over %gs' % budget())
                 sys.stdout.flush()
+            # os._exit stops save() wherever it is, so a half-written temp file would stay in the plan.
+            try:
+                _tmp(plan).unlink()
+            except OSError:
+                pass
             os._exit(0)
         return 0
     if not (plan / 'tasks.tsv').is_file():

package/scripts/forge_dataset/harvest.py

@@ -43,7 +43,7 @@ WATCHED = (
     'verification.status', 'verification.exit', 'verification.command', 'verification.log',
     'verification.retry.log', 'verification.flaky', 'verification.coverage.txt',
     'verification.jev.json', 'verification.runs.tsv', 'verification.commit', 'verification.fingerprint',
-    'results.tsv',
+    'verification.lanes.json', 'results.tsv',
 )
 
 _VERDICT = re.compile(r'^FORGE_VERDICT: (PASS|FAIL)([ \t]+(—|–|-)( .*)?)?[ \t]*$')
@@ -544,8 +544,11 @@ def verification_events(attempt: Path, out: Path, rc, prov: dict, src: str = 'li
     repo = first_line(plan / 'repo')
     command = first_line(out / 'verification.command') if made('verification.command') else None
     jev = read_json(out / 'verification.jev.json') if made('verification.jev.json') else None
+    lanes = made('verification.lanes.json')
     if not command:
         source = None
+    elif lanes:
+        source = 'lanes:plan' if (plan / 'verify.d').is_dir() else 'lanes:repo'
     elif (plan / 'verify_cmd').exists():
         source = 'flag'
     elif jev and jev.get('discovered'):
@@ -559,9 +562,12 @@ def verification_events(attempt: Path, out: Path, rc, prov: dict, src: str = 'li
         for line in (text(out / 'verification.runs.tsv') or '').splitlines():
             cells = line.split('\t')
             if len(cells) >= 5 and cells[0].isdigit():
-                runs.append(dict(n=int(cells[0]), kind=cells[1], exit=as_int(cells[2]),
-                                 tree_changed=cells[3] == '1', duration_s=as_int(cells[4]),
-                                 log=cells[5] if len(cells) > 5 else None))
+                run = dict(n=int(cells[0]), kind=cells[1], exit=as_int(cells[2]),
+                           tree_changed=cells[3] == '1', duration_s=as_int(cells[4]),
+                           log=cells[5] if len(cells) > 5 else None)
+                if len(cells) >= 8:          # a lane's run: n kind exit tree_changed duration_s log lane started
+                    run['lane'] = cells[6] or None
+                runs.append(run)
     elif command and made('verification.log'):
         runs.append(dict(n=1, kind='initial', exit=as_int(first_line(out / 'verification.exit')),
                          tree_changed=None, duration_s=None, log='verification.log'))

package/scripts/forge_verify_reports.py

@@ -0,0 +1,225 @@
+"""Test reports for forge-verify.py: JUnit XML and TAP 12-14. Stdlib only, offline.
+
+``scan(directory, since)`` reads every ``*.xml`` and ``*.tap`` under a lane's report
+directory that was written at or after ``since`` (an epoch float: the start of that run),
+recursively and without following symlinks, and returns the lane's ``tests``:
+
+    {"pass", "fail", "skip", "error",
+     "failing": [{"name", "suite", "file", "message", "excerpt", "known", "log_line"}],
+     "reports": [<paths relative to the directory>], "notes": [...], "truncated": bool}
+
+plus what a baseline verdict needs and the display does not keep: every failing name (not
+only the first 200) and whether the evidence is complete. A report that cannot be used
+(too big, unreadable, not parseable, a DOCTYPE or ENTITY declaration) is skipped with a
+note and makes the evidence incomplete; nothing here raises on a hostile file.
+"""
+from __future__ import annotations
+
+import os
+import re
+import stat
+import xml.etree.ElementTree as ET
+
+FILE_MAX = 32 << 20
+FAILING_MAX = 200
+MESSAGE_MAX = 500
+EXCERPT_MAX = 2000
+LOG_SEARCH_MAX = 64 << 20
+
+ANSI = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-Z\\-_]')
+# Refused before parsing: no external entities, no entity expansion bombs.
+DECLARATIONS = re.compile(rb'<!\s*(?:DOCTYPE|ENTITY)', re.I)
+TAP_LINE = re.compile(r'(not\s+)?ok\b\s*(\d+)?\s*(?:-\s*)?(.*)$')
+TAP_PLAN = re.compile(r'1\.\.(\d+)\b')
+TAP_BAIL = re.compile(r'Bail out!\s*(.*)$')
+TAP_DIRECTIVE = re.compile(r'\s*(?<!\\)#\s*(SKIP|TODO)\b', re.I)
+
+
+def clean(text: str | None, cap: int) -> str:
+    return ANSI.sub('', text or '').strip()[:cap]
+
+
+def first_line(text: str | None) -> str:
+    for line in (text or '').splitlines():
+        if line.strip():
+            return line.strip()
+    return ''
+
+
+def junit_cases(data: bytes) -> list[dict]:
+    """[{name, suite, outcome, message, excerpt}] from a JUnit XML document; ValueError when unusable."""
+    # The declaration check reads bytes, so only an ASCII-compatible encoding is let through:
+    # UTF-16 or UTF-32 (a byte-order mark, or NUL bytes) could hide a DOCTYPE from it.
+    if data[:2] in (b'\xff\xfe', b'\xfe\xff') or b'\x00' in data:
+        raise ValueError('only UTF-8 reports are read')
+    if DECLARATIONS.search(data):
+        raise ValueError('DOCTYPE or ENTITY declarations are refused')
+    try:
+        root = ET.fromstring(data)
+    except ET.ParseError as error:
+        raise ValueError('not well-formed XML (%s)' % error)
+    if root.tag not in ('testsuite', 'testsuites'):
+        raise ValueError('no testsuite or testsuites element')
+    parents = {child: parent for parent in root.iter() for child in parent}
+    cases = []
+    for case in root.iter('testcase'):
+        name, classname = case.get('name') or '?', case.get('classname')
+        suite = parents.get(case)
+        problem = case.find('failure')
+        outcome = 'fail'
+        if problem is None:
+            problem, outcome = case.find('error'), 'error'
+        if problem is None:
+            outcome = 'skip' if case.find('skipped') is not None else 'pass'
+        entry = {'name': '%s.%s' % (classname, name) if classname else name,
+                 'suite': suite.get('name') if suite is not None and suite.tag == 'testsuite' else None,
+                 'outcome': outcome, 'message': '', 'excerpt': ''}
+        if problem is not None:
+            text = problem.text or ''
+            entry['message'] = problem.get('message') or first_line(text)
+            parts = [text] + [(case.findtext(tag) or '') for tag in ('system-out', 'system-err')]
+            entry['excerpt'] = '\n'.join(p.strip('\n') for p in parts if p.strip())
+        cases.append(entry)
+    return cases
+
+
+def tap_cases(data: bytes) -> tuple[list[dict], str | None]:
+    """Top-level test points of a TAP stream, each with its YAML diagnostic block if any, and
+    why the stream is incomplete (a `Bail out!`, no plan, or fewer or more points than its
+    plan), else None. An incomplete stream's points still count, but cannot prove a baseline."""
+    try:
+        text = data.decode('utf-8')
+    except UnicodeDecodeError:
+        raise ValueError('not UTF-8 text')
+    cases: list[dict] = []
+    block: list[str] | None = None
+    planned = bail = None
+    for line in text.splitlines():
+        if block is not None:
+            if line.strip() == '...':
+                if cases:
+                    diag = '\n'.join(block)
+                    cases[-1]['excerpt'] = diag
+                    found = re.search(r'^\s*message:\s*(.*)$', diag, re.M)
+                    if found:
+                        cases[-1]['message'] = found.group(1).strip().strip('\'"')
+                block = None
+            else:
+                block.append(line)
+            continue
+        if line.strip() == '---' and line.startswith(' ') and cases:
+            block = []
+            continue
+        stop = TAP_BAIL.match(line)
+        if stop:
+            bail = stop.group(1).strip() or 'no reason given'
+            break
+        plan = TAP_PLAN.match(line)
+        if plan and planned is None:
+            planned = int(plan.group(1))
+            continue
+        match = TAP_LINE.match(line)     # top level only: subtests are indented
+        if not match:
+            continue
+        description = match.group(3)
+        directive = TAP_DIRECTIVE.search(description)
+        name = description[:directive.start()].strip() if directive else description.strip()
+        if directive:
+            outcome = 'skip'
+        else:
+            outcome = 'fail' if match.group(1) else 'pass'
+        cases.append({'name': name or 'test %s' % (match.group(2) or len(cases) + 1), 'suite': None,
+                      'outcome': outcome, 'message': '', 'excerpt': ''})
+    if not cases:
+        raise ValueError('no TAP test points')
+    if bail is not None:
+        return cases, 'it bailed out (%s)' % clean(bail, MESSAGE_MAX)
+    if planned is None:
+        return cases, 'it has no plan line (1..N)'
+    if planned != len(cases):
+        return cases, 'its plan says %d tests and it has %d' % (planned, len(cases))
+    return cases, None
+
+
+def scan(directory: str, since: float) -> tuple[dict, list[str], bool]:
+    """(tests, every failing name, whether the evidence is complete) for one lane run."""
+    tests = {'pass': 0, 'fail': 0, 'skip': 0, 'error': 0, 'failing': [], 'reports': [],
+             'notes': [], 'truncated': False}
+    names: list[str] = []
+    complete = True
+    found = []
+    for top, dirs, files in os.walk(directory, followlinks=False):
+        dirs.sort()
+        for name in sorted(files):
+            if name.endswith(('.xml', '.tap')):
+                found.append(os.path.join(top, name))
+    for path in found:
+        rel = os.path.relpath(path, directory)
+        try:
+            info = os.lstat(path)
+        except OSError:
+            continue
+        if not stat.S_ISREG(info.st_mode) or info.st_mtime < since:
+            continue          # a symlink, or a report an earlier run left
+        try:
+            if info.st_size > FILE_MAX:
+                raise ValueError('larger than %d MB' % (FILE_MAX >> 20))
+            with open(path, 'rb') as handle:
+                data = handle.read(FILE_MAX + 1)
+            if len(data) > FILE_MAX:
+                raise ValueError('larger than %d MB' % (FILE_MAX >> 20))
+            cases, problem = (junit_cases(data), None) if rel.endswith('.xml') else tap_cases(data)
+        except (OSError, ValueError, RecursionError) as error:
+            tests['notes'].append('%s skipped: %s' % (rel, error))
+            complete = False
+            continue
+        tests['reports'].append(rel)
+        if problem:
+            tests['notes'].append('%s is incomplete: %s' % (rel, problem))
+            complete = False
+        for case in cases:
+            tests[case['outcome']] += 1
+            if case['outcome'] not in ('fail', 'error'):
+                continue
+            names.append(case['name'])
+            if len(tests['failing']) >= FAILING_MAX:
+                tests['truncated'] = True
+                continue
+            tests['failing'].append({'name': clean(case['name'], MESSAGE_MAX), 'suite': case['suite'],
+                                     'file': rel, 'message': clean(case['message'], MESSAGE_MAX),
+                                     'excerpt': clean(case['excerpt'], EXCERPT_MAX),
+                                     'known': False, 'log_line': None})
+    return tests, names, complete
+
+
+def log_lines(log: str, names: list[str]) -> dict:
+    """{name: 1-based number of the first log line containing it}, within the first 64 MB."""
+    wanted = sorted({n for n in names if n}, key=len, reverse=True)
+    if not wanted:
+        return {}
+    pattern = re.compile('|'.join(re.escape(n) for n in wanted))
+    found: dict = {}
+    read = 0
+    try:
+        with open(log, 'rb') as handle:
+            for number, raw in enumerate(handle, 1):
+                read += len(raw)
+                if read > LOG_SEARCH_MAX:
+                    break
+                line = raw.decode('utf-8', 'replace')
+                if not pattern.search(line):    # one search per line; most lines name no test
+                    continue
+                for name in wanted:
+                    if name not in found and name in line:
+                        found[name] = number
+                if len(found) == len(wanted):
+                    break
+    except OSError:
+        pass
+    return found
+
+
+def load_baseline(path: str) -> set:
+    """Known failing test names: one per line, blank lines and # comments ignored."""
+    with open(path, encoding='utf-8', errors='replace') as handle:
+        return {line.strip() for line in handle if line.strip() and not line.strip().startswith('#')}

package/ui/README.md

@@ -12,9 +12,12 @@ blocks on a plate, with cables between them and lamps that light up as things ha
   dependency cables, and ends in a *finish* line of three nodes: integration, verification
   and the whole-run **final review**. *Execution* lays the same work out on wall-clock time:
   one block per attempt, in the lane it really ran in, with its dwarf and review inside, the
-  attempts that have not started projected by forge-schedule's own rules, and a strip of how
-  many slots were busy against `--max-parallel`. Switching between the two moves each task
-  from its wave to where it ran.
+  attempts that have not started projected by forge-schedule's own rules, a strip of how
+  many slots were busy against `--max-parallel`, and the verification's lanes and reruns on
+  rows of their own. Switching between the two moves each task from its wave to where it ran.
+- **Verification as it happens:** the step each lane is in, its last line, the failing tests
+  first (new or known, from the lanes' test reports) with a link into the log at each one, and
+  every lane's output a click away.
 - **Click to open.** A task opens to show its attempts, each with its dwarf, its reviewer and
   any shadow reviews, linked; an escalation is a labelled cable between attempts. The
   inspector beside the stage shows what launched each step and what came of it: the exact
@@ -178,6 +181,18 @@ keep one meaning everywhere (amber running, green merged or passed, red failed,
   with the gate decision, Clef predictions against outcomes, and its prompts; for one
   execution, its model, phases, tokens, exact prompt and final message; for the final
   review, its verdict, findings, report, prompt and history.
+- **Verification (its finish node):** while it runs, how long it has run, the step a lane
+  announced (`::forge step name=… state=start`) and for how long, the last line it printed, a
+  warning after five minutes with no output, and the live output of the lane that printed last
+  (choose a lane to follow it instead). Once it failed, the failing tests come first, grouped by
+  lane and marked *new* or *known* (from the lane's JUnit or TAP reports and its baseline), each
+  with its message, an excerpt and *open in log* at its line; a lane without reports shows the
+  end of its log, and the step it ended in is named. A table lists every lane with its state
+  (flaky and known marked), time, runs and test counts; a lane's name opens its output. A
+  changed checkout is called out. A single verify command shows the newest of its logs live.
+  In *Execution*, the verification gets rows after the tasks: one per lane, with a bar per run
+  and reruns marked, or a single command's runs back to back (dashed: only their durations are
+  recorded). The finish node counts the lanes and reruns and lights one lamp per lane.
 - **Files (the run toolbar's third mode):** a heatmap of files (grouped and collapsible by
   directory) against tasks (grouped by wave): planned files outlined, touched files filled by how
   many attempts changed them, drift hatched, merge conflicts ringed, and a dot where another task

package/ui/docs/api.md

@@ -43,7 +43,9 @@ written), and the reader must never raise because of one.
 | `shadow_qa` | a comma list of specs |
 | `timeout` | seconds |
 | `verification.status` | `PASS`, `FAIL` or another word; `verification.command` the command; `verification.log` its output |
-| `verification.runs.tsv` | `n<TAB>kind<TAB>exit<TAB>tree_changed<TAB>duration_s<TAB>logfile` per run of the command; `kind` is `initial`, `rerun` or `jev_retry`, `exit` may be empty, `tree_changed` is `0` or `1` |
+| `verification.runs.tsv` | `n<TAB>kind<TAB>exit<TAB>tree_changed<TAB>duration_s<TAB>logfile` per run of the command; `kind` is `initial`, `rerun` or `jev_retry`, `exit` may be empty, `tree_changed` is `0` or `1`. A lanes verification adds `<TAB>lane<TAB>started` (an epoch) |
+| `verification.retry.log` | the output of a deterministic or Jev rerun of a single command |
+| `verification.lanes.json` | a lanes verification's live and final state, written by `scripts/forge-verify.py` (read up to 4 MB); `verification.lane.<name>.log` and `verification.lane.<name>.run-<k>.log` each lane's output |
 | `verification.commit` | one line, the commit verified |
 | `max_parallel`, `escalate_after`, `verify_retries` | one integer each |
 | `qa_threshold` | `P0`..`P3`, the least severe finding that still blocks; `final_qa` the final review's QA spec |
@@ -167,6 +169,7 @@ def execution_transcript(path, name, source='plan') -> dict | None # an Executio
 def fractal_detail(path, task_id, source='plan') -> dict | None    # a TaskFractal
 def fractal_node(path, task_id, node, source='plan') -> dict | None       # a FractalNodeDetail
 def fractal_node_diff(path, task_id, node, source='plan') -> dict | None  # a DiffDetail
+def verification_log(path, lane=None, k=None, first=None, retry=False, source='plan') -> dict | None  # a VerificationLog
 
 class Catalog:
     def __init__(self, plans=(), roots=(), registry=True, registry_dir=None): ...
@@ -178,7 +181,8 @@ class Catalog:
     def diff(self, key, task_id, which='current'), execution_diff(self, key, name),
         final_review_diff(self, key, name), files(self, key), findings(self, key), transcript(self, key, name),
         fractal(self, key, task_id),
-        fractal_node(self, key, task_id, node), fractal_node_diff(self, key, task_id, node) -> dict | None
+        fractal_node(self, key, task_id, node), fractal_node_diff(self, key, task_id, node),
+        verification_log(self, key, lane=None, k=None, first=None) -> dict | None
     def models(self) -> dict            # Models, over every run
     def locate(self, key) -> tuple[str, str] | None  # (run dir, 'plan' | 'solo'); None for an unknown key
 ```
@@ -225,6 +229,10 @@ its last 20,000 behind a leading `…`. Their `truncated` lists the field names
 | `GET /api/runs/<key>/files` | `200 Files` |
 | `GET /api/runs/<key>/findings` | `200 RunFindings` |
 | `GET /api/runs/<key>/executions/<qa-or-dwarf>/transcript` | `200 ExecutionTranscript` |
+| `GET /api/runs/<key>/verification/log[/lines/<n>]` | `200 VerificationLog` of `verification.log`: its tail, or with `/lines/<n>` a range |
+| `GET /api/runs/<key>/verification/retry/log[/lines/<n>]` | `200 VerificationLog` of `verification.retry.log`, a single command's rerun |
+| `GET /api/runs/<key>/verification/lanes/<lane>/log[/lines/<n>]` | `200 VerificationLog` of `verification.lane.<lane>.log` |
+| `GET /api/runs/<key>/verification/lanes/<lane>/runs/<k>/log[/lines/<n>]` | `200 VerificationLog` of `verification.lane.<lane>.run-<k>.log` |
 | `GET /api/models` | `200 Models` |
 | `GET /api/health` | `200 {"app": "forge-ui", "version", "build", "pid", "controls": true\|false}`; `controls` is `false` in read-only mode |
 | `GET /api/runs/<key>/controls` | `200` the `forge control show --json` object of a plan run (see Controls); `404` for a solo or unknown run |
@@ -251,11 +259,14 @@ Fractal node endpoints for an unknown node or a task whose Fractal state is not
 diff that is not there yet is a `200` with `available: false`. An execution is looked for only where executions are written:
 `attempts/`, `tasks/<id>/attempts/`, `shadow/<id>/<a>-<k>/attempts/`,
 `final-review-*/attempts/` and `integrate-*/final-review/attempts/`, in that order; a dir
-that resolves outside the run dir (a symlinked `attempts/`, say) is ignored.
+that resolves outside the run dir (a symlinked `attempts/`, say) is ignored. The verification
+log endpoints answer `404` for a solo run, a `<lane>` not matching `[A-Za-z0-9][A-Za-z0-9_-]{0,63}`
+(so no `..`), a `<k>` or `<n>` that is not a positive decimal without a leading zero, and a log
+that is missing, a symlink or not a regular file.
 
 JSON responses: `Content-Type: application/json; charset=utf-8`,
 `Cache-Control: no-store`. The page loads `style.css`, `runmodel.js`, `graph.js`, `gantt.js`,
-`panel.js`, `map3d.js` and `app.js` by those relative names, and `scene3d.js` and `iso.js`
+`verifyview.js`, `panel.js`, `map3d.js` and `app.js` by those relative names, and `scene3d.js` and `iso.js`
 (ES modules) only when the 3D view is used.
 
 Every response carries `X-Content-Type-Options: nosniff`, `Referrer-Policy: no-referrer`
@@ -342,8 +353,14 @@ Every RunSummary field, plus:
                "final_qa": "opus"},
   "verification": {"status": "PASS", "command": "python3 -m unittest", "log_tail": "...",
                    "runs": [{"n": 1, "kind": "initial", "exit": 0, "tree_changed": false,
-                             "duration_s": 12.3, "log": "verification.run-1.log"}],
-                   "commit": "9c1e4b7a..."},
+                             "duration_s": 12.3, "log": "verification.lane.unit.run-1.log",
+                             "lane": "unit", "started": 1760622000.5}],
+                   "commit": "9c1e4b7a...",
+                   "state": "done", "mode": "lanes", "live": null, "mutated": false,
+                   "lanes": [VerificationLane, ...],
+                   "failing": [{"lane": "e2e", "name": "checkout.Pay.test_refund", "suite": "checkout",
+                                "file": "e2e.xml", "message": "expected 200, got 500", "excerpt": "...",
+                                "known": false, "log_line": 12}]},
   "waves": [["reader", "page", "graph"], ["server"], ["readme"]],
   "tasks": [Task, ...],
   "timeline": {"started": 1760620000.25, "ended": null, "runs": [Execution], "verifications": [Execution]},
@@ -369,7 +386,53 @@ Every RunSummary field, plus:
 - `verification` is `null` when none of `verification.status`, `.command`, `.log` exists. A
   solo run's `verification` is always `null`. `runs` comes from `verification.runs.tsv`
   (malformed rows are warned and skipped; an empty `exit` is `null`), `commit` is the first
-  line of `verification.commit` or `null`.
+  line of `verification.commit` or `null`. A run's lanes runs carry `lane` and `started`; a
+  single command's have `null` for both. A row of neither 6 nor 8 columns is warned and skipped.
+- `verification.state` is `running` when the run's newest `verification` execution (the newest
+  of `timeline.verifications`) is running, `done` when it has ended, and `null` when the run has
+  none. While it runs, `status` is `RUNNING`, whatever a `verification.status` left by an earlier
+  verification says.
+- `verification.mode` is `lanes` when `verification.lanes.json` exists and was written at or
+  after the newest verification execution started (or there is none), else `single`. A lanes file
+  an earlier verification left is not read.
+- `verification.live` is `null` unless `state` is `running`; then `{started, step, last_line,
+  last_output_at, idle_s, log}`. `started` is the execution's start. In `single` mode the live log is
+  whichever of `verification.log` and `verification.retry.log` was modified last: `step` is
+  `{name, started: null}` for the last `::forge step name=<x> state=start` in its last 64 KB
+  that no later `state=end` closed (a line holds no time), `last_line` its last non-empty line
+  without ANSI codes (at most 300 characters), and `last_output_at` its mtime. In `lanes` mode
+  they are the running lane's that printed last, with its recorded `step.started`. `idle_s` is
+  now minus `last_output_at` (minus `started` before any output). `log` names the live log in
+  `single` mode (`verification.log` or `verification.retry.log`; `null` before either exists) and is
+  `null` in `lanes` mode.
+- `verification.lanes` is `null` in `single` mode and when the lanes file is unreadable, not a
+  JSON object, has no `lanes` list or is over 4 MB (each warned; the request never fails). In
+  `lanes` mode it is every lane:
+
+  ```
+  VerificationLane = {"name": "e2e", "command": "npm run test:e2e", "group": "browser", "first": false,
+                      "retries": 2, "options": {"retries": 2, "reports": "reports"},
+                      "state": "fail", "exit": 1, "started": 1760622000.5, "ended": 1760622070.0,
+                      "duration_s": 69.5, "idle_s": null,
+                      "runs": [{"k": 1, "kind": "initial", "started": 1760622000.5, "ended": 1760622030.0,
+                                "exit": 1, "log": "verification.lane.e2e.run-1.log"}],
+                      "step": {"name": "browser", "started": 1760622010.0} | null,
+                      "last_line": "2 failed, 30 passed", "last_output_at": 1760622069.8,
+                      "log": "verification.lane.e2e.log",
+                      "tests": {"pass": 30, "fail": 2, "skip": 1, "error": 0, "failing": [...],
+                                "reports": ["e2e.xml"], "notes": [], "truncated": false} | null}
+  ```
+
+  `state` is one of `waiting running pass fail flaky known skipped interrupted`; `duration_s`
+  runs from `started` to `ended`, or to now while the lane runs; `idle_s` is now minus
+  `last_output_at` for a running lane, else `null`. `tests` is `null` when the lane wrote no
+  report; its `failing` holds at most 200 entries (`truncated` says more failed), a `message` at
+  most 500 characters and an `excerpt` at most 2,000; `log_line` is the 1-based line of the
+  lane's log that first names the test, or `null`.
+- `verification.failing` is every lane's `tests.failing`, in lane order, each with its `lane`:
+  at most 200 entries, `[]` in `single` mode.
+- `verification.mutated` is `true` when the checkout changed under the verification (the lanes
+  file's `mutated`, or in `single` mode a run with `tree_changed`): reruns after it were skipped.
 - `timeline.started` is the earliest `started` of every execution the detail shows (tasks
   and their shadows, the listed final reviews, the run's own); `ended` is the latest `ended`,
   or `null` while any of them is `running` or when none has ended. `runs` and
@@ -773,6 +836,30 @@ characters …` (`<key>_cut` holds n). A log over 32 MB is read from its end; pa
 4,000,000 characters the middle is left out (a `gap` event, `truncated: true`). Parses are kept by
 the log's `(path, size, mtime_ns)` and the session file's.
 
+## VerificationLog
+
+The run's own verification logs, read by path (an integration's lane logs are not served yet):
+
+```
+{"name": "verification.lane.e2e.log", "size": 220000, "mtime": 1760622070.0,
+ "tail": "line 14044\n...", "tail_first_line": 14044, "truncated": true}
+```
+
+`tail` is the file's last 64 KB, from the first line that starts inside them (when no line
+does, from the first whole character), decoded as UTF-8 with invalid bytes replaced.
+`tail_first_line` is the 1-based number of the tail's first line, found by counting the newlines
+before it, or `null` when that would read past 64 MB. `truncated` says the tail is not the whole
+file. With `/lines/<n>`:
+
+```
+{"name": "verification.log", "size": 11000, "first_line": 700, "lines": ["line 00700", ...],
+ "truncated": false}
+```
+
+up to 400 lines from line `n` (1-based), each without its line ending and cut to 4,000
+characters (ending `…`). Lines past the end give `lines: []`. `truncated` is `true` when more
+lines follow, a line was cut, or reaching them took more than 64 MB of reading.
+
 ## Files
 
 `GET /api/runs/<key>/files`: who planned, changed, drifted into or conflicted on each file.
@@ -942,13 +1029,15 @@ difficulty paths deps model` each), `decomposition_history` (its newest 20, each
   it (server.api_get): `/api/runs` (this run only), the RunDetail, every TaskDetail, every
   ExecutionDetail the RunDetail names, every FinalReviewDetail, Files, RunFindings, and the
   Fractal trees and nodes. With `texts=1` also every diff the page can open (each task's diff and its `sources`,
-  each final review's, each Fractal node's) and every `dwarf` and `qa` ExecutionTranscript (at most 2 MB each).
+  each final review's, each Fractal node's), every `dwarf` and `qa` ExecutionTranscript (at most 2 MB each),
+  and the VerificationLog tail of `verification.log`, of each lane's log and of each lane run's log
+  (under their tail paths; line ranges are not stored, so the page shows the tail).
 - `texts=0` (the default) nulls these keys wherever they hold a string, list or object: `prompt
   last request log_tail dwarf_tail qa_tail qa_last qa_input tasks_summary known_issues texts text
-  planning_last approach retry_findings dwarf_input error`, and stores no diff and no transcript. A
-  finding keeps its `summary`.
-- `paths=0` (the default) makes every string that is a whole absolute path (but `/api/…` and
-  `url`s) its last component, and replaces the run dir, the repo and the home dir inside other
+  planning_last approach retry_findings dwarf_input error last_line message excerpt tail lines`, and stores no
+  diff and no transcript. A finding keeps its `summary`, a failing test its `name`.
+- `paths=0` (the default) makes every string that is a whole absolute path (but `/api/…`, `url`s
+  and `command`s) its last component, and replaces the run dir, the repo and the home dir inside other
   strings with their last component (`~` for home).
 - At most 1 MB of JSON per diff and 25 MB in all; what did not fit is listed in `meta.omitted`
   (`path`, `bytes`, `why`). `meta`: `format` `forge-ui-export`, `version` 1, `app`, `key`,
@@ -1061,7 +1150,14 @@ for them only unknown keys and runs the catalog does not hold are `404`. Every w
 `forge control show <plan dir> --json`: `200` with its object (`plan`, `run_id`, `live`, `fractal`,
 `run` (the plan-wide controls), `tasks` (per task: `status`, `dwarf`, `qa`, `qa_threshold`,
 `retry_budget`, `retries_used`, `pin`, `escalation_stop`, `escalation_next`) and `log`). No token
-is needed to read it.
+is needed to read it. Of `run` the page also reads `max_parallel` (1–64 or `null`), `infra_backoff`
+(the shape, `exp:60` when unset), `escalate_effort` (`on`/`off`), `drained` and `stopping`
+(`graceful`, `now` or `null`) while live, and `infra_paused` while live: one
+`{scope, class, harness, model, tasks, cycle, until, exhausted}` per paused scope (`scope` is
+`all`, `harness:<h>` or `model:<h>/<m>`; `until` an epoch, `exhausted` true once its retries are
+spent). `log` is the newest `control.log` lines first; a scheduler's answer to a queued request
+is a line with `actor: "scheduler"`, the request's `request` id, `outcome` (`applied` or
+`refused`) and `reason`.
 
 ### `POST /api/runs/<key>/controls`
 
@@ -1076,10 +1172,22 @@ known or a spec runs is for the CLI to say (its `400` or `409` comes back as abo
 | `{"action": "set", "key": "retry-budget", "value": "inherit", "task": "reader"}` | `control set <dir> retry-budget inherit --task reader --actor ui --json` |
 | `{"action": "pin", "task": "graph", "spec": "sol:high:codex"}` | `control pin <dir> graph sol:high:codex --actor ui --json` |
 | `{"action": "unpin", "task": "graph"}` | `control unpin <dir> graph --actor ui --json` |
+| `{"action": "drain"}` | `control drain <dir> --actor ui --json` |
+| `{"action": "resume"}` | `control resume <dir> --actor ui --json` |
+| `{"action": "stop"}` (or `"now": false`) | `control stop <dir> --actor ui --json` |
+| `{"action": "stop", "now": true}` | `control stop <dir> --now --actor ui --json` |
+| `{"action": "retry", "task": "graph"}` | `control retry <dir> graph --actor ui --json` |
+| `{"action": "retry", "task": "graph", "dwarf": "sol:high:codex"}` | `control retry <dir> graph --dwarf sol:high:codex --actor ui --json` |
 
 `value` is always a string. `task` is optional on `set`, and only for the keys the CLI allows it
-for. A success is `200 {"ok": true, "changed": {...}, "applies": "...", "controls": {...}}`, where
-`controls` is the `GET` object after the change.
+for; the page sets `max-parallel`, `infra-backoff` (`none` drops it) and `escalate-effort`
+(`on`/`off`) plan-wide. `now` is a boolean. `add` is not exposed: a row is added to `tasks.tsv` by
+hand, so its CLI is the place to queue it. A success is
+`200 {"ok": true, "changed": {...}, "applies": "...", "controls": {...}}`, where `controls` is the
+`GET` object after the change; `drain`, `resume` and `stop` answer the same way and are refused
+(`409`) when no scheduler is live. A `retry` only queues: `200 {"queued": true, "request": "<id>"}`.
+Its refusals before queueing (not live, a MERGED task, a spec that fails doctor) are the CLI's
+`409`/`400`; the scheduler applies or refuses it later, in the `log` line with that `request` id.
 
 ### `GET /api/runs/<key>/issues`
 
@@ -1180,9 +1288,10 @@ Plain scripts unless noted; each defines one global and none parses API text as
 
 | File | Global | What it is |
 |---|---|---|
-| `runmodel.js` | `ForgeRunModel` | pure data, no DOM: the plate layout (archivision's `plate()`), the atlas, plan and execution models, forge-schedule's rules projected forward, the critical path, concurrency, the state of every node at a time `t` (replay), filters. Also a CommonJS module, for `node --test` |
-| `gantt.js` | `ForgeGantt` | `render(container, run, {now, onSelect, sel, replay, critical, ghosts})`: attempts on wall-clock time in lanes; `strip(container, run, {now, replay})`: busy attempts and model calls against `max_parallel` |
+| `runmodel.js` | `ForgeRunModel` | pure data, no DOM: the plate layout (archivision's `plate()`), the atlas, plan and execution models, forge-schedule's rules projected forward, the critical path, concurrency, the state of every node at a time `t` (replay, verification lanes included), the finish node's verification LEDs (`verifyLeds`: one per lane in lanes mode, else one per run), filters. Also a CommonJS module, for `node --test` |
+| `gantt.js` | `ForgeGantt` | `render(container, run, {now, onSelect, sel, replay, critical, ghosts})`: attempts on wall-clock time in lanes; `strip(container, run, {now, replay})`: busy attempts and model calls against `max_parallel`. Under the attempts, the finish rows: `verify` (single-command verifications; the newest split into its runs back to back from its start, dashed because that placement is approximate), a `verify · <lane>` row per lane of a lanes verification (each run and rerun at its own time; a run with no exit is UNKNOWN once it ended, never FAIL) and `final review`. A lane or run item opens the verification inspector |
 | `panel.js` | `ForgePanel` | `render(host, ctx)`: the inspector for a run, task, execution, final review, verification, integration or Fractal node; fetches the lazy endpoints once per node; `ctx.diffHash(spec)` links into the diff viewer, `ctx.onFractal(task, TaskFractal)` hands a task's tree to the stage |
+| `verifyview.js` | `ForgeVerify` | `render(host, {run, now, fetchJSON, rerender})`: the verification inspector panel.js delegates to: while it runs, the elapsed time, step, last line and a flag past 5 minutes without output, and a live tail of the followed lane's log (or the single log) read again each poll; when it failed, the failing tests first, grouped by lane with new or known, each opening its lane's log at its line (in an export, the tail); a lanes table (state, time, reruns, counts). Pure helpers (`model`, `laneRow`, `groupFailing`, `idleWarning`, `openStep`, `tailLane`, `logPath`, `linesPath`, `tailLines`) are a CommonJS module too |
 | `diffview.js` | `ForgeDiff` | `render(host, {run, spec, fetchJSON, rerender, href, closeHref})`: a DiffDetail with its findings under their lines; `spec` is `{kind: task, task, source, file}`, `{kind: final, name, file}` or `{kind: fractal, task, node, file}` |
 | `convo.js` | `ForgeConvo` | `render(host, {run, spec: {task, exec}, brief, live, fetchJSON, rerender, diffHash, closeHref})`: an ExecutionTranscript as a timeline, with search, the errors and, while the execution runs, a read of its log every 2.5 s |
 | `files.js` | `ForgeFiles` | `matrix(data, run, opts)` (pure: Files as rows of dirs and files against task columns by wave) and `render(host, data, {run, sel, onSelectTask, diffHref, prefs, savePrefs})`: the run's Files mode. Also a CommonJS module |

package/ui/forge_ui/control.py

@@ -199,9 +199,13 @@ def validate_body(body):
     action = body.get('action')
     shapes = {'set': ({'action', 'key', 'value', 'task'}, ('key', 'value')),
               'pin': ({'action', 'task', 'spec'}, ('task', 'spec')),
-              'unpin': ({'action', 'task'}, ('task',))}
+              'unpin': ({'action', 'task'}, ('task',)),
+              'retry': ({'action', 'task', 'dwarf'}, ('task',)),
+              'drain': ({'action'}, ()),
+              'resume': ({'action'}, ()),
+              'stop': ({'action', 'now'}, ())}
     if not isinstance(action, str) or action not in shapes:
-        raise BadBody('action must be one of set, pin, unpin')
+        raise BadBody('action must be one of ' + ', '.join(shapes))
     allowed, required = shapes[action]
     extra = sorted(set(body) - allowed)
     if extra:
@@ -209,6 +213,12 @@ def validate_body(body):
     tail = [_text(body, name) for name in required]
     if action == 'set' and 'task' in body:
         tail += ['--task', _text(body, 'task')]
+    if action == 'retry' and 'dwarf' in body:
+        tail += ['--dwarf', _text(body, 'dwarf')]
+    if action == 'stop' and 'now' in body:
+        if not isinstance(body['now'], bool):
+            raise BadBody('now must be true or false')
+        tail += ['--now'] if body['now'] else []
     return action, tail