@giamme/forge 0.11.1 → 0.12.0

package/ui/forge_ui/static/controls.js

@@ -25,7 +25,13 @@
     'verify-retries': { label: 'Verify retries', run: 'at the next verification', planOnly: true },
     'retry-budget': { label: 'Retry budget', run: 'at the next retry decision', task: 'at the next retry decision', live: true },
     'infra-retries': { label: 'Infra retries', run: 'at the next retry decision', planOnly: true, live: true },
+    'max-parallel': { label: 'Max parallel', run: 'at the next dispatch', planOnly: true, live: true },
+    'infra-backoff': { label: 'Infra backoff', run: 'at the next infrastructure wait', planOnly: true },
+    'escalate-effort': { label: 'Effort step', run: 'at the next retry decision', planOnly: true },
   };
+  const BACKOFF_FORMS = 'exp:<base>[,<cap>], linear:<start>,<step>[,<cap>] or list:<a>,<b>,… (seconds)';
+  // A task status after which "Retry now" applies (or the policy stopped it).
+  const RETRYABLE = ['FAIL', 'UNKNOWN', 'TIMEOUT', 'ERROR', 'INTERRUPTED', 'BLOCKED'];
 
   const obj = v => v && typeof v === 'object' && !Array.isArray(v) ? v : {};
   const num = v => typeof v === 'number' && Number.isFinite(v) ? v : null;
@@ -80,6 +86,16 @@
       return LEVELS.includes(text) || text === 'none' || (task && text === 'inherit') ? { ok: true, value: text } : bad('P0, P1, P2, P3 or none');
     }
     if (task && key === 'retry-budget' && text === 'inherit') return { ok: true, value: 'inherit' };
+    if (key === 'max-parallel') {
+      return /^[0-9]{1,2}$/.test(text) && Number(text) >= 1 && Number(text) <= 64 ? { ok: true, value: String(Number(text)) } : bad('a whole number, 1 to 64');
+    }
+    if (key === 'escalate-effort') return text === 'on' || text === 'off' ? { ok: true, value: text } : bad('on or off');
+    if (key === 'infra-backoff') {
+      // The shape itself is the CLI's to check; here only what can never be one. "default" drops the setting.
+      if (text === 'default' || text === 'none') return { ok: true, value: 'none' };
+      return text && !/[\s\u0000-\u001f\u007f]/.test(text) && !text.startsWith('-') && text.length <= MAX_FIELD
+        ? { ok: true, value: text } : bad('a backoff shape: ' + BACKOFF_FORMS);
+    }
     return key === 'timeout' ? secondsOf(text) : countOf(text);
   }
 
@@ -92,11 +108,15 @@
     }
     if (action === 'pin') return { action: 'pin', task: a.task, spec: a.spec };
     if (action === 'unpin') return { action: 'unpin', task: a.task };
+    if (action === 'drain' || action === 'resume') return { action };
+    if (action === 'stop') return a && a.now ? { action: 'stop', now: true } : { action: 'stop' };
+    if (action === 'retry') return a.dwarf ? { action: 'retry', task: a.task, dwarf: a.dwarf } : { action: 'retry', task: a.task };
     throw new Error('unknown action ' + action);
   }
 
   const shown = (key, value, task) => key === 'timeout' ? `${minutesOf(Number(value))} min`
-    : key === 'qa-threshold' && value === 'none' && task ? 'none (no threshold for this task)' : value;
+    : key === 'qa-threshold' && value === 'none' && task ? 'none (no threshold for this task)'
+    : key === 'infra-backoff' && value === 'none' ? 'the default (exp:60)' : value;
   const inTask = task => task ? ` for task \`${task}\`` : '';
 
   // A draft change → {ok, body, confirm} or {ok: false, error}. change: {kind: 'set', key, input, task?} |
@@ -120,6 +140,21 @@
       return c.task ? { ok: true, body: bodyFor('unpin', { task: c.task }),
         confirm: `Remove the pin on task \`${c.task}\`? Its next retry goes back to the escalation policy` } : bad('no task');
     }
+    if (c.kind === 'drain') return { ok: true, body: bodyFor('drain'), confirm: 'Drain the run? Running tasks finish; nothing new starts until you resume' };
+    if (c.kind === 'resume') return { ok: true, body: bodyFor('resume'), confirm: 'Resume the run? The scheduler starts tasks again' };
+    if (c.kind === 'stop') {
+      return { ok: true, body: bodyFor('stop', { now: !!c.now }), confirm: c.now
+        ? 'Stop the run now? Running tasks are interrupted, as Ctrl-C does; the run ends with no verification'
+        : 'Stop the run once its running tasks finish? Nothing new starts, and the run ends with no verification (unless every task has merged by then)' };
+    }
+    if (c.kind === 'retry') {
+      if (!c.task) return bad('no task');
+      const named = String(c.alias === undefined || c.alias === null ? '' : c.alias).trim();
+      const s = named ? specOf(c) : { ok: true, spec: null };
+      if (!s.ok) return s;
+      return { ok: true, body: bodyFor('retry', { task: c.task, dwarf: s.spec }),
+        confirm: `Retry task \`${c.task}\` now${s.spec ? ` on \`${s.spec}\`` : ''}? It is queued into the live run` };
+    }
     return bad('unknown change');
   }
 
@@ -129,6 +164,7 @@
     if (d.changed && String(rec.previous === undefined || rec.previous === null ? '' : rec.previous) === String(rec.value === undefined || rec.value === null ? '' : rec.value)) {
       return 'Nothing changed: it was already so.';
     }
+    if (d.queued === true) return 'Queued: the scheduler applies or refuses it at its next turn.';
     return typeof d.applies === 'string' && d.applies ? `Saved. Applies ${d.applies}.` : 'Saved.';
   }
 
@@ -139,8 +175,46 @@
       qa: LEVELS.includes(run.qa_threshold) ? run.qa_threshold : 'none',
       timeout: num(run.timeout), timeoutMin: minutesOf(run.timeout),
       escalateAfter: num(run.escalate_after), verifyRetries: num(run.verify_retries),
-      retryBudget: num(run.retry_budget), infraBudget: num(run.infra_budget) };
+      retryBudget: num(run.retry_budget), infraBudget: num(run.infra_budget),
+      maxParallel: num(run.max_parallel), infraBackoff: typeof run.infra_backoff === 'string' ? run.infra_backoff : null,
+      escalateEffort: run.escalate_effort === 'on' ? 'on' : 'off', drained: run.drained === true,
+      stopping: run.stopping === 'graceful' || run.stopping === 'now' ? run.stopping : null,
+      pauses: Array.isArray(run.infra_paused) ? run.infra_paused.filter(x => x && typeof x === 'object') : [] };
+  }
+  // "3m05s" to a moment, from `now` (epoch seconds); "now" once it has come.
+  function countdown(until, now) {
+    const s = num(until) === null || num(now) === null ? null : Math.ceil(until - now);
+    if (s === null) return null;
+    if (s <= 0) return 'now';
+    return s < 60 ? `${s}s` : s < 3600 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, '0')}s` : `${Math.floor(s / 3600)}h${String(Math.floor(s % 3600 / 60)).padStart(2, '0')}m`;
+  }
+  // One line per infrastructure pause (`run.infra_paused`): what is paused, why, its tasks, the wait, and the time to
+  // its end, or "budget used up" once its retries are spent.
+  function pauseLines(controls, now) {
+    return runValues(controls).pauses.map(x => {
+      const scope = String(x.scope || '');
+      const what = scope === 'all' ? 'every task' : scope.startsWith('model:') ? `model ${x.model || scope.slice(6)}${x.harness ? ' on ' + x.harness : ''}`
+        : scope.startsWith('harness:') ? `harness ${x.harness || scope.slice(8)}` : scope || 'a scope';
+      const tasks = Array.isArray(x.tasks) && x.tasks.length ? x.tasks.map(String).join(', ') : null;
+      const left = x.exhausted === true ? 'budget used up' : countdown(x.until, now);
+      return { scope, exhausted: x.exhausted === true, text: [what, x.class ? String(x.class) : null, tasks ? 'tasks ' + tasks : null,
+        num(x.cycle) !== null ? `wait ${x.cycle}` : null, left === null ? null : left === 'budget used up' ? left : left === 'now' ? 'resuming' : `resumes in ${left}`]
+        .filter(Boolean).join(' · ') };
+    });
   }
+  // Where a queued retry stands: {state: 'queued'} until the scheduler's line with the same request id is in the log,
+  // then {state: 'applied'} or {state: 'refused', reason}. Only the request id matches a line.
+  function requestState(request, log) {
+    const line = (Array.isArray(log) ? log : []).find(e => e && e.actor === 'scheduler' && e.request === request);
+    if (!request || !line) return { state: 'queued' };
+    return line.outcome === 'applied' ? { state: 'applied' } : { state: 'refused', reason: typeof line.reason === 'string' ? line.reason : null };
+  }
+  const requestText = st => st.state === 'refused' ? `refused${st.reason ? ': ' + st.reason : ''}` : st.state;
+  // Whether "Retry now" shows for a task: live only, after it ended without passing or once the policy stopped it.
+  const canRetry = (controls, id) => {
+    const c = obj(controls), v = taskValues(c, id);
+    return c.live === true && v.known && v.status !== 'MERGED' && (RETRYABLE.includes(v.status) || v.stop);
+  };
   // A task's controls with the effective value of each: an override wins, else the run's value is inherited.
   function taskValues(controls, id) {
     const c = obj(controls), t = obj(obj(c.tasks)[id]), run = runValues(c);
@@ -211,7 +285,8 @@
 
   /* ---------- the page: drawing ---------- */
 
-  const store = { enabled: null, runs: new Map(), models: null, modelsAsked: false, lists: new Set(), recent: new Map() };
+  // requests: `<run key>|<task>` → the id of the retry this page queued, until the scheduler's line answers it
+  const store = { enabled: null, runs: new Map(), models: null, modelsAsked: false, lists: new Set(), recent: new Map(), requests: new Map() };
   const RECENT_MS = 9000;                                    // how long a result line stays after the section is rebuilt
   const STALE_MS = 20000;
   let seq = 0;
@@ -284,8 +359,9 @@
     return seg;
   }
 
-  // Ask, send, show the answer. A success redraws every row from the controls the answer carries.
-  async function commit(view, r, change, fields, button) {
+  // Ask, send, show the answer. A success redraws every row from the controls the answer carries; `done` gets a
+  // success's answer (a queued retry keeps its request id).
+  async function commit(view, r, change, fields, button, done) {
     const u = ui();
     if (view.inflight) { u.outcome(r.out, { ok: false, status: 409, error: 'busy' }); return; }
     clearOutcome(view, r);
@@ -299,8 +375,10 @@
     const res = await u.post(path(view.run.key), plan.body);
     view.inflight = false; view.section.removeAttribute('aria-busy');
     remember(view, r, res, savedText(res.data));
+    // sent: the fields are no longer an edit, also when the answer (a queued request) carries no controls
+    if (res.ok) for (const f of fields) delete view.draft[f];
+    if (res.ok && done) done(res.data);
     if (res.ok && obj(res.data).controls && res.data.controls.run) {
-      for (const f of fields) delete view.draft[f];
       accept(view.run.key, res.data.controls, view.run.updated);
       draw(view, res.data.controls);
     } else sync(view);
@@ -332,19 +410,74 @@
   }
 
   function liveLamp(view, c) {
-    const u = ui(), live = c.live === true, box = u.el('p', undefined, 'ctl-live');
+    const u = ui(), live = c.live === true, box = u.el('p', undefined, 'ctl-live'), v = runValues(c);
     box.append(u.el('i', undefined, 'lamp ' + (live ? 'st-running' : 'st-pending')),
-      u.el('span', live ? 'live' : 'not running', 'plate'), u.el('span', live ? 'a scheduler is running this plan' : 'no scheduler is running this plan', 'muted'));
+      u.el('span', live ? 'live' : 'not running', 'plate'));
+    if (live && v.stopping) box.append(u.el('span', v.stopping === 'now' ? 'stopping now' : 'stopping', 'tag ctl-state ctl-stopping'));
+    else if (live && v.drained) box.append(u.el('span', 'drained', 'tag ctl-state ctl-drained'));
+    box.append(u.el('span', !live ? 'no scheduler is running this plan' : v.stopping ? 'the run ends once its running tasks are done'
+      : v.drained ? 'running tasks finish; nothing new starts until resumed' : 'a scheduler is running this plan', 'muted'));
+    return box;
+  }
+
+  // Drain or Resume, Stop and Stop now: live only, each asked first.
+  function runKeys(view, c) {
+    const u = ui(), v = runValues(c), r = shell(view, 'run', 'Run', false);
+    const keys = u.el('div', undefined, 'ctl-keys');
+    const key = (label, change, title, cls) => {
+      const b = u.el('button', label, cls); b.type = 'button'; b.title = title;
+      b.addEventListener('click', () => commit(view, r, change, [], b));
+      keys.append(b);
+    };
+    if (v.drained) key('Resume', { kind: 'resume' }, 'Start tasks again');
+    else key('Drain', { kind: 'drain' }, 'Start nothing new; running tasks finish and the run waits');
+    if (!v.stopping) {
+      key('Stop', { kind: 'stop' }, 'Let running tasks finish, then end the run (no verification, unless every task merged)');
+      key('Stop now', { kind: 'stop', now: true }, 'Interrupt running tasks and end the run', 'ui-danger');
+    }
+    r.control.append(keys);
+    help(r, v.stopping ? 'Stopping: nothing more to ask of the run.' : 'Stop lets running tasks finish; Stop now interrupts them.');
+    return r.box;
+  }
+
+  // One line per infrastructure pause; its countdown ticks from the page's clock between polls.
+  function pauseBox(view, c) {
+    const u = ui(), box = u.el('div', undefined, 'ctl-pauses');
+    box.append(u.el('span', 'Paused on infrastructure', 'ctl-label'));
+    const ul = u.el('ul', undefined, 'ctl-pause-list');
+    const paint = () => {
+      const lines = pauseLines(c, Date.now() / 1000);
+      ul.replaceChildren(...lines.map(l => {
+        const li = u.el('li', undefined, 'ctl-pause' + (l.exhausted ? ' ctl-exhausted' : ''));
+        li.append(u.el('i', undefined, 'lamp ' + (l.exhausted ? 'st-fail' : 'st-running')), u.el('span', l.text, 'ctl-pause-text'));
+        return li;
+      }));
+    };
+    paint();
+    if (view.ticker) clearInterval(view.ticker);
+    view.ticker = setInterval(() => { if (box.isConnected) paint(); else { clearInterval(view.ticker); view.ticker = null; } }, 1000);
+    box.append(ul);
     return box;
   }
 
+  // An on/off setting as a two-key bank.
+  function switchRow(view, c, o) {
+    const r = shell(view, o.key, o.label, false), perm = o.perm;
+    r.control.append(bank(r, [{ value: 'off', label: 'Off' }, { value: 'on', label: 'On' }], o.current, perm.enabled, (value, button) => {
+      if (value !== o.current) commit(view, r, { kind: 'set', key: o.key, input: value }, [], button);
+    }));
+    help(r, o.help);
+    if (!perm.enabled) why(r, perm.why);
+    return r.box;
+  }
+
   // An integer or minutes setting: a field and a Set key; Enter sets.
   function countRow(view, c, o) {
     const u = ui(), r = shell(view, o.key, o.label, true), perm = o.perm;
     const form = u.el('form', undefined, 'ctl-set');
     form.noValidate = true;
     const input = u.el('input');
-    input.type = 'text'; input.inputMode = o.decimal ? 'decimal' : 'numeric'; input.autocomplete = 'off'; input.spellcheck = false;
+    input.type = 'text'; input.inputMode = o.text ? 'text' : o.decimal ? 'decimal' : 'numeric'; input.autocomplete = 'off'; input.spellcheck = false;
     input.id = r.id + '-in'; input.placeholder = o.placeholder || ''; input.disabled = !perm.enabled;
     const name = o.key + (o.task ? '@' + o.task : '');
     input.value = field(view, name, o.value === null ? '' : String(o.value));
@@ -403,6 +536,10 @@
     const v = runValues(c), body = ui().el('div', undefined, 'ctl-body');
     const perm = key => canEdit(c, key);
     body.append(liveLamp(view, c));
+    if (c.live === true) {
+      body.append(runKeys(view, c));
+      if (v.pauses.length) body.append(pauseBox(view, c));
+    }
     body.append(levelRow(view, c, { key: 'qa-threshold', label: 'QA threshold', current: v.qa, perm: perm('qa-threshold'),
       help: v.qa === 'none' ? 'The least severe finding that still blocks: no threshold is set.'
         : `The least severe finding that still blocks: ${v.qa} (${LEVELS.slice(0, LEVELS.indexOf(v.qa) + 1).join(', ')} block).`, noneTitle: 'No threshold' }));
@@ -416,6 +553,13 @@
       help: 'Retries a task may use, run-wide.' }));
     body.append(countRow(view, c, { key: 'infra-retries', label: 'Infra retries', unit: 'retries', value: v.infraBudget, placeholder: 'not set', perm: perm('infra-retries'),
       help: 'Retries for infrastructure failures (auth, quota, network), run-wide.' }));
+    body.append(countRow(view, c, { key: 'max-parallel', label: 'Max parallel', unit: 'tasks', value: v.maxParallel, placeholder: 'as started', perm: perm('max-parallel'),
+      help: 'Tasks running at once, 1 to 64. A lower limit lets running tasks finish; it never stops one.' }));
+    body.append(countRow(view, c, { key: 'infra-backoff', label: 'Infra backoff', text: true, value: v.infraBackoff, placeholder: 'exp:60', perm: perm('infra-backoff'),
+      help: `The wait between infrastructure retries: ${BACKOFF_FORMS}. "default" goes back to exp:60.` }));
+    body.append(switchRow(view, c, { key: 'escalate-effort', label: 'Effort step', current: v.escalateEffort, perm: perm('escalate-effort'),
+      help: v.escalateEffort === 'on' ? 'On: at the top tier, a task is retried once more at the next effort its model accepts before it stops.'
+        : 'Off: at the top tier, a task that used up its escalation stops with a split suggestion.' }));
     body.append(changeLog(c));
     return body;
   }
@@ -435,9 +579,66 @@
       inherit: rb.mode === 'inherit' ? 'pressed' : 'ready', perm: canEdit(c, 'retry-budget', id),
       help: `${rb.mode === 'inherit' ? (rb.effective === null ? 'Inherits the run\'s budget.' : `Inherits ${rb.effective} from the run.`) : `${rb.own} for this task.`} ${used}.` }));
     if (!canEdit(c, 'pin', id).hidden) body.append(pinRow(view, c, v));
+    if (canRetry(c, id) || store.requests.has(view.run.key + '|' + id)) body.append(retryRow(view, c, v));
     return body;
   }
 
+  // The implementer picker the pin and Retry now share: alias (suggestions from the models seen in runs, free text too),
+  // effort, optional harness. Drafts live under `<prefix>.alias|effort|harness`; read() gives the change's parts.
+  function modelFields(view, r, prefix) {
+    const u = ui(), alias = u.el('input'), harness = u.el('input'), list = u.el('datalist');
+    alias.type = 'text'; harness.type = 'text';
+    for (const i of [alias, harness]) { i.autocomplete = 'off'; i.spellcheck = false; }
+    alias.setAttribute('list', r.id + '-models');
+    list.id = r.id + '-models'; fillList(list);
+    if (!store.models) trackList(list);
+    alias.value = field(view, prefix + '.alias', ''); harness.value = field(view, prefix + '.harness', '');
+    alias.addEventListener('input', () => touch(view, prefix + '.alias', alias.value));
+    harness.addEventListener('input', () => touch(view, prefix + '.harness', harness.value));
+    const captioned = (caption, control, cls) => { const l = u.el('label', undefined, 'filter ' + cls); l.append(u.el('span', caption), control); return l; };
+    const effortCaption = u.el('span', 'Effort'), effortBox = u.el('div', undefined, 'filter ctl-effort');
+    effortCaption.id = r.id + '-effort';
+    const effortBank = bank(r, EFFORTS.map(e => ({ value: e, label: e })), field(view, prefix + '.effort', 'default'), true, value => {
+      touch(view, prefix + '.effort', value);
+      for (const b of effortBank.children) b.setAttribute('aria-pressed', String(b.dataset.value === value));
+    });
+    effortBank.setAttribute('aria-labelledby', effortCaption.id);
+    effortBox.append(effortCaption, effortBank);
+    loadModels();
+    return { nodes: [captioned('Model alias', alias, 'ctl-alias'), list, captioned('Harness (optional)', harness, 'ctl-harness'), effortBox],
+      read: () => ({ alias: alias.value, effort: view.draft[prefix + '.effort'] || 'default', harness: harness.value }),
+      fields: [prefix + '.alias', prefix + '.effort', prefix + '.harness'] };
+  }
+
+  // Retry now: queue a retry of a failed or stopped task into the live run, on its policy's model or one named here.
+  function retryRow(view, c, v) {
+    const u = ui(), r = shell(view, 'retry', 'Retry now', false), id = v.id, hold = view.run.key + '|' + id;
+    const request = store.requests.get(hold);
+    if (request) {
+      const st = requestState(request, c.log);
+      const line = u.el('p', undefined, 'ctl-request ctl-request-' + st.state);
+      line.append(u.el('i', undefined, 'lamp ' + (st.state === 'applied' ? 'st-pass' : st.state === 'refused' ? 'st-fail' : 'st-running')),
+        u.el('span', requestText(st)));
+      r.status.append(line);
+    }
+    if (!canRetry(c, id)) return r.box;
+    const form = u.el('form', undefined, 'ctl-pin ctl-retry');
+    form.noValidate = true;
+    const picker = modelFields(view, r, 'retry');
+    const go = u.el('button', 'Retry now', 'primary'); go.type = 'submit';
+    form.append(...picker.nodes, go);
+    form.addEventListener('submit', e => {
+      e.preventDefault();
+      commit(view, r, Object.assign({ kind: 'retry', task: id }, picker.read()), picker.fields, go, data => {
+        if (data && data.queued === true && typeof data.request === 'string') store.requests.set(hold, data.request);
+        show(view);
+      });
+    });
+    r.control.append(form);
+    help(r, 'Leave the model empty for the escalation policy\'s choice.');
+    return r.box;
+  }
+
   function pinRow(view, c, v) {
     const u = ui(), r = shell(view, 'pin', 'Pin next retry to', false), id = v.id;
     const now = u.el('div', undefined, 'ctl-pin-now');
@@ -455,39 +656,21 @@
     }
     if (v.stop) {
       const stop = u.el('p', undefined, 'ctl-stop');
-      stop.append(u.el('i', undefined, 'lamp st-fail'), u.el('span', 'policy stopped: split or pin a model'));
+      // "Retry now" is only offered while the run is live
+      stop.append(u.el('i', undefined, 'lamp st-fail'), u.el('span', canRetry(c, id) ? 'policy stopped: split, pin a model, or Retry now' : 'policy stopped: split or pin a model'));
       r.status.append(stop);
     }
-    // alias (suggestions from the models seen in runs, free text too), effort, optional harness
     const form = u.el('form', undefined, 'ctl-pin');
     form.noValidate = true;
-    const alias = u.el('input'), harness = u.el('input'), list = u.el('datalist');
-    alias.type = 'text'; harness.type = 'text';
-    for (const i of [alias, harness]) { i.autocomplete = 'off'; i.spellcheck = false; }
-    alias.setAttribute('list', r.id + '-models');
-    list.id = r.id + '-models'; fillList(list);
-    if (!store.models) trackList(list);
-    alias.value = field(view, 'pin.alias', ''); harness.value = field(view, 'pin.harness', '');
-    alias.addEventListener('input', () => touch(view, 'pin.alias', alias.value));
-    harness.addEventListener('input', () => touch(view, 'pin.harness', harness.value));
-    const captioned = (caption, control, cls) => { const l = u.el('label', undefined, 'filter ' + cls); l.append(u.el('span', caption), control); return l; };
-    const effortCaption = u.el('span', 'Effort'), effortBox = u.el('div', undefined, 'filter ctl-effort');
-    effortCaption.id = r.id + '-effort';
-    const effortBank = bank(r, EFFORTS.map(e => ({ value: e, label: e })), field(view, 'pin.effort', 'default'), true, value => {
-      touch(view, 'pin.effort', value);
-      for (const b of effortBank.children) b.setAttribute('aria-pressed', String(b.dataset.value === value));
-    });
-    effortBank.setAttribute('aria-labelledby', effortCaption.id);
-    effortBox.append(effortCaption, effortBank);
+    const picker = modelFields(view, r, 'pin');
     const go = u.el('button', 'Pin', 'primary'); go.type = 'submit';
-    form.append(captioned('Model alias', alias, 'ctl-alias'), list, captioned('Harness (optional)', harness, 'ctl-harness'), effortBox, go);
+    form.append(...picker.nodes, go);
     form.addEventListener('submit', e => {
       e.preventDefault();
-      commit(view, r, { kind: 'pin', task: id, alias: alias.value, effort: view.draft['pin.effort'] || 'default', harness: harness.value }, ['pin.alias', 'pin.effort', 'pin.harness'], go);
+      commit(view, r, Object.assign({ kind: 'pin', task: id }, picker.read()), picker.fields, go);
     });
     r.control.append(form);
     help(r, 'Applies at the next full retry, not a QA-only resume.');
-    loadModels();
     return r.box;
   }
   function fillList(list) {
@@ -615,5 +798,6 @@
   }
 
   return { LEVELS, EFFORTS, KEYS, specOf, countOf, secondsOf, minutesOf, valueFor, bodyFor, prepare, savedText, effective, canEdit, logLine, agoText,
-    aliasesOf, scopeOf, typing, isEditing, keepEditing, redrawDue, mountRun, mountTask, hold };
+    aliasesOf, scopeOf, typing, isEditing, keepEditing, redrawDue, countdown, pauseLines, requestState, requestText, canRetry,
+    mountRun, mountTask, hold };
 });

package/ui/forge_ui/static/gantt.js

@@ -20,10 +20,13 @@ window.ForgeGantt = (() => {
     const X = RM.execModel(run, { now, ghosts });
     const all = X.items.concat(X.extra);
     if (!all.length) { const p = document.createElement('p'); p.className = 'fg-empty'; p.textContent = 'Nothing has run yet: no attempt timings are recorded for this run.'; container.append(p); return; }
-    const width = Math.max(640, container.clientWidth || 900), plot = width - LABEL - 12;
-    const xs = t => LABEL + (t - X.t0) / (X.t1 - X.t0) * plot;
-    const laneOf = it => typeof it.lane === 'number' ? it.lane : X.lanes + (it.kind === 'verification' ? 0 : X.finishLanes.length > 1 ? 1 : 0);
+    const laneOf = it => typeof it.lane === 'number' ? it.lane : X.lanes + Number(String(it.lane).slice(1));
     const rows = X.lanes + X.finishLanes.length;
+    const laneName = i => { const fl = X.finishLanes[i - X.lanes]; return i < X.lanes ? `lane ${i + 1}${i >= X.cap ? ' !' : ''}` : fl.label || fl.name.toLowerCase(); };
+    // the label column fits the longest row name ("verify · <lane>" can be long)
+    const lw = Math.max(LABEL, Math.ceil(12 + 6.6 * Math.max(...Array.from({ length: rows }, (_, i) => laneName(i).length))));
+    const width = Math.max(640, container.clientWidth || 900), plot = width - lw - 12;
+    const xs = t => lw + (t - X.t0) / (X.t1 - X.t0) * plot;
     const height = TOP + rows * LANE + 8;
     const root = svg('svg', { class: 'gantt', viewBox: `0 0 ${width} ${height}`, width, height, role: 'group', 'aria-label': 'Attempts on wall-clock time' });
     for (const r of X.ruler) {
@@ -32,8 +35,8 @@ window.ForgeGantt = (() => {
     }
     for (let i = 0; i < rows; i++) {
       const over = i < X.lanes && i >= X.cap, y = TOP + i * LANE;
-      svg('rect', { class: 'gt-lane' + (over ? ' gt-over' : ''), x: LABEL, y: y + 2, width: plot, height: LANE - 4, rx: 4 }, root);
-      const name = i < X.lanes ? `lane ${i + 1}${over ? ' !' : ''}` : X.finishLanes[i - X.lanes].name.toLowerCase();
+      svg('rect', { class: 'gt-lane' + (over ? ' gt-over' : ''), x: lw, y: y + 2, width: plot, height: LANE - 4, rx: 4 }, root);
+      const name = laneName(i);
       const t = svg('text', { class: 'gt-lane-label' + (over ? ' gt-over' : ''), x: 6, y: y + LANE / 2 + 4 }, root); t.textContent = name;
       if (over) svg('title', {}, t).textContent = `More attempts overlapped than --max-parallel ${X.cap} allows: separate run invocations, or a resumed run.`;
     }
@@ -41,19 +44,21 @@ window.ForgeGantt = (() => {
     for (const it of all) {
       const y = TOP + laneOf(it) * LANE + 5, x = xs(it.start), w = Math.max(3, xs(it.end) - x);
       const future = replay !== null && it.start > replay;
-      const g = svg('g', { class: `gt-item ${stc(it.status)}${it.ghost ? ' gt-ghost' : ''}${future ? ' gt-future' : ''}${critSet.has(it.task) && !it.ghost ? ' gt-crit' : ''}${sel && (sel.exec === it.id || (sel.kind === 'task' && sel.id === it.task)) ? ' gt-sel' : ''}`, tabindex: 0, role: 'button' }, root);
+      const picked = sel && (sel.exec === it.id || (sel.kind === 'task' && sel.id === it.task) || (sel.kind === 'verification' && it.kind === 'verification'));
+      const g = svg('g', { class: `gt-item ${stc(it.status)}${it.ghost ? ' gt-ghost' : ''}${it.approx ? ' gt-approx' : ''}${it.rerun ? ' gt-rerun' : ''}${future ? ' gt-future' : ''}${critSet.has(it.task) && !it.ghost ? ' gt-crit' : ''}${picked ? ' gt-sel' : ''}`, tabindex: 0, role: 'button' }, root);
       svg('rect', { class: 'gt-box', x, y, width: w, height: LANE - 10, rx: 3 }, g);
       for (const s of it.segs) {
         const sx = xs(s.start), sw = Math.max(1.5, xs(s.end ?? it.end) - sx);
         svg('rect', { class: `gt-seg gt-${s.role} ${stc(s.status)}`, x: sx, y: y + LANE - 17, width: sw, height: 5, rx: 1.5 }, g);
       }
-      const label = it.kind === 'verification' ? 'verify' : it.kind === 'review' ? 'review' : `${it.task}${it.attempt ? ' A' + it.attempt : ''}`;
+      const label = it.vlane ? (it.rerun ? `rerun ${it.k - 1}` : `run ${it.k}`) : it.kind === 'verification' ? (it.rerun ? 'rerun' : 'verify') : it.kind === 'review' ? 'review' : `${it.task}${it.attempt ? ' A' + it.attempt : ''}`;
       if (w > label.length * 6.4 + 6) svg('text', { class: 'gt-label', x: x + 4, y: y + 11 }, g).textContent = label;
-      svg('title', {}, g).textContent = `${label} · ${it.status} · ${RM.fmtDur(it.end - it.start)}${it.ghost ? ' (projected)' : ''}`;
+      svg('title', {}, g).textContent = it.title ? `${it.title} · ${it.status} · ${RM.fmtDur(it.end - it.start)}`
+        : `${label} · ${it.status} · ${RM.fmtDur(it.end - it.start)}${it.ghost ? ' (projected)' : ''}`;
       const pick = () => {
         if (!onSelect) return;
         if (it.kind === 'review') onSelect({ kind: 'final', name: it.review });
-        else if (it.kind === 'verification') onSelect({ kind: 'execution', task: null, exec: it.id });
+        else if (it.kind === 'verification') onSelect(it.vlane || it.approx ? { kind: 'verification' } : { kind: 'execution', task: null, exec: it.id });
         else if (it.ghost || String(it.id).startsWith('span:')) onSelect({ kind: 'task', id: it.task });
         else onSelect({ kind: 'execution', task: it.task, id: it.task, exec: it.id });
       };

package/ui/forge_ui/static/index.html

@@ -25,6 +25,7 @@
   <script src="graph.js" defer></script>
   <script src="gantt.js" defer></script>
   <script src="controls.js" defer></script>
+  <script src="verifyview.js" defer></script>
   <script src="panel.js" defer></script>
   <script src="diffview.js" defer></script>
   <script src="convo.js" defer></script>

package/ui/forge_ui/static/panel.js

@@ -559,10 +559,11 @@ window.ForgePanel = (() => {
     head.append(el('h2', 'Verification'), el('div', 'finish', 'insp-kind'), chip(v && v.status || 'pending'));
     host.append(head);
     if (!v) { muted(host, 'Verification has not run: it runs after every task is merged into the integration branch.'); return; }
+    if (window.ForgeVerify) window.ForgeVerify.render(host, ctx);     // live state, lanes and failing tests
     fields(host, [['Command', v.command, true], ['Commit', v.commit, true]]);
     if (list(v.runs).length) {
       const ul = el('ul', undefined, 'plain');
-      for (const r of v.runs) ul.append(el('li', `#${display(r.n)} ${display(r.kind)} · exit ${display(r.exit)} · ${RM.fmtDur(r.duration_s)}${r.tree_changed ? ' · changed the tree' : ''}`));
+      for (const r of v.runs) ul.append(el('li', `#${display(r.n)} ${display(r.kind)}${r.lane ? ' · ' + r.lane : ''} · exit ${display(r.exit)} · ${RM.fmtDur(r.duration_s)}${r.tree_changed ? ' · changed the tree' : ''}`));
       section(host, 'Runs').append(ul);
     }
     for (const x of list(run.timeline && run.timeline.verifications)) { const s = { kind: 'execution', task: null, exec: x.id }; host.append(el('p'), link(`Timings of the verification started ${time(x.started)} →`, ctx.hash(s), () => ctx.onSelect(s))); }

package/ui/forge_ui/static/runmodel.js

@@ -287,7 +287,9 @@
     return {
       integration: { status: integrationStatus(tasks.map(t => t.status)),
         merged, total: tasks.length, base: ig && ig.base || null, accepted: ig && ig.accepted || null },
-      verification: { status: v && v.status ? String(v.status).toUpperCase() : 'PENDING', command: v && v.command || null, runs: list(v && v.runs).length },
+      verification: { status: v && v.state === 'running' ? 'RUNNING' : v && v.status ? String(v.status).toUpperCase() : 'PENDING',
+        command: v && v.command || null, runs: list(v && v.runs).length, mode: v && v.mode === 'lanes' && list(v.lanes).length ? 'lanes' : 'single',
+        lanes: list(v && v.lanes).map(l => ({ name: l.name, state: l.state })), reruns: list(v && v.runs).filter(r => r.kind !== 'initial').length },
       final: { status: !frObj ? (run.settings && run.settings.final_qa ? 'PENDING' : 'NONE') : finalStatus(frObj),
         requested: !!(fr || run.settings && run.settings.final_qa), spec: frObj && frObj.spec || run.settings && run.settings.final_qa || null, findings: frObj && frObj.findings || null,
         count: list(run.final_reviews).length,
@@ -475,20 +477,68 @@
     }
     return items;
   }
+  // A verification run's status: exit 0 passed, any other exit failed, no exit is RUNNING while it runs and UNKNOWN after.
+  const runStatus = (exit, running) => exit === 0 ? 'PASS' : num(exit) !== null ? 'FAIL' : running ? 'RUNNING' : 'UNKNOWN';
+  const LANE_STATUS = { flaky: 'FLAKY', known: 'KNOWN' };
+  // The finish rows' verification items, each with its `row`: in lanes mode a row per lane of the newest verification,
+  // one item per run of it (the newest verification's earlier executions stay boxes on a VERIFY row); in single mode
+  // the newest verification split into its runs, back to back from its start (`approx`: the execution also covers
+  // discovery and fingerprinting, and a run's duration its fingerprint), else one box per execution.
+  function verifyItems(run, now) {
+    const v = run && run.verification, execs = list(run && run.timeline && run.timeline.verifications).filter(e => num(e.started) !== null);
+    const newest = execs.reduce((a, e) => !a || e.started >= a.started ? e : a, null);
+    const lanes = v && v.mode === 'lanes' ? list(v.lanes).filter(l => l && l.name) : [];
+    const out = [];
+    for (const e of execs) {
+      if (e === newest && lanes.length) continue;
+      const runs = e === newest ? list(v && v.runs) : [];
+      const running = e.state === 'running';
+      if (!runs.length) {
+        out.push({ id: e.id, kind: 'verification', row: 'verification', start: e.started, end: execEnd(e, now) || e.started,
+          status: running ? 'RUNNING' : e.exit_code === 0 ? 'PASS' : e.exit_code == null ? 'UNKNOWN' : 'FAIL', segs: [] });
+        continue;
+      }
+      let t = e.started;
+      runs.forEach((r, i) => {
+        const d = Math.max(0, num(r.duration_s) || 0), rerun = r.kind !== 'initial';
+        out.push({ id: `vrun:${e.id}:${r.n ?? i + 1}`, kind: 'verification', row: 'verification', exec: e.id, approx: true, rerun,
+          start: t, end: t + d, status: runStatus(num(r.exit), false), segs: [],
+          title: `verify · run ${r.n ?? i + 1}${rerun ? ' (' + String(r.kind).replace('_', ' ') + ')' : ''} · approximate placement` });
+        t += d;
+      });
+      // runs.tsv gains a row when a run ends: a verification still going is in a run that is not listed yet
+      if (running && num(now) !== null && now > t) out.push({ id: `vrun:${e.id}:now`, kind: 'verification', row: 'verification', exec: e.id, approx: true,
+        start: t, end: now, status: 'RUNNING', segs: [], title: 'verify · the current run · approximate placement' });
+    }
+    for (const l of lanes) {
+      const runs = list(l.runs);
+      runs.forEach((r, i) => {
+        const s = num(r.started); if (s === null) return;
+        const last = i === runs.length - 1, running = last && l.state === 'running' && num(r.ended) === null;
+        const end = num(r.ended) ?? (running && num(now) !== null ? Math.max(s, now) : s);
+        const status = last && LANE_STATUS[l.state] ? LANE_STATUS[l.state] : runStatus(num(r.exit), running);
+        const k = num(r.k) ?? i + 1, rerun = r.kind === 'rerun';
+        const notes = [l.group ? 'group ' + l.group : null, l.first ? 'runs first' : null].filter(Boolean);
+        out.push({ id: `vlane:${l.name}:${k}`, kind: 'verification', row: 'vlane:' + l.name, vlane: l.name, k, rerun, exec: newest ? newest.id : null,
+          start: s, end, running, status, segs: [], title: `verify · ${l.name} · ${rerun ? 'rerun ' + (k - 1) : 'run ' + k}${notes.length ? ' · ' + notes.join(' · ') : ''}` });
+      });
+    }
+    return { items: out, lanes: lanes.map(l => l.name) };
+  }
   // scale: {t0, t1} of a previous model, kept while every block still fits inside it, so a live run does not
   // change the time axis (and so every block) on each poll
   function execModel(run, { now = null, ghosts = true, maxParallel = null, scale = null } = {}) {
     const items = execItems(run, now);
     const cap = Math.max(1, num(maxParallel) || num(run && run.settings && run.settings.max_parallel) || 3);
-    const live = items.some(i => i.running) || list(run && run.tasks).some(t => t.status === 'RUNNING');
+    const live = items.some(i => i.running) || list(run && run.tasks).some(t => t.status === 'RUNNING')
+      || !!(run && run.verification && run.verification.state === 'running');
     const proj = ghosts && run && run.kind !== 'solo' ? projectSchedule(run, { now: now !== null ? now : undefined, maxParallel: cap }) : {};
     for (const [id, p] of Object.entries(proj)) if (p.ghost && p.start !== null) items.push({ id: 'ghost:' + id, task: id, ghost: true, start: p.start, end: p.end, status: 'PENDING', segs: [] });
-    const extra = [];
-    for (const v of list(run && run.timeline && run.timeline.verifications)) if (num(v.started) !== null) extra.push({ id: v.id, kind: 'verification', start: v.started, end: execEnd(v, now) || v.started, status: v.state === 'running' ? 'RUNNING' : v.exit_code === 0 ? 'PASS' : v.exit_code == null ? 'UNKNOWN' : 'FAIL', segs: [] });
+    const verify = verifyItems(run, now), extra = verify.items;
     for (const r of list(run && run.final_reviews)) {
       const s = num(r.started) ?? num(r.at); if (s === null) continue;
       const e = num(r.ended) ?? (r.state === 'running' && num(now) !== null ? now : s + 1);
-      extra.push({ id: 'review:' + r.name, kind: 'review', review: r.name, start: s, end: Math.max(s, e), status: r.state === 'running' ? 'RUNNING' : r.verdict || 'UNKNOWN', findings: r.findings || null, segs: [] });
+      extra.push({ id: 'review:' + r.name, kind: 'review', row: 'review', review: r.name, start: s, end: Math.max(s, e), status: r.state === 'running' ? 'RUNNING' : r.verdict || 'UNKNOWN', findings: r.findings || null, segs: [] });
     }
     const all = items.concat(extra);
     if (!all.length) return { items: [], extra: [], lanes: 0, cap, overLimit: false, t0: null, t1: null, k: 1, px: 900, py: 300, ruler: [], live, nowX: null };
@@ -515,13 +565,15 @@
     const x = t => EXEC_PAD + (t - t0) * k;
     for (const it of items) { it.x = x(it.start); it.w = Math.max(22, (it.end - it.start) * k); it.y = RULER + it.lane * LANE_D + (LANE_D - BLOCK_D) / 2; it.d = BLOCK_D; }
     const fy = RULER + lanes * LANE_D + 24;
-    const finishLanes = [['verification', 'VERIFY'], ['review', 'FINAL REVIEW']].filter(([k]) => extra.some(e => e.kind === k));
-    finishLanes.forEach(([kind], i) => extra.filter(e => e.kind === kind).forEach(e => { e.lane = 'f' + i; e.x = x(e.start); e.w = Math.max(22, (e.end - e.start) * k); e.y = fy + i * LANE_D + (LANE_D - BLOCK_D) / 2; e.d = BLOCK_D; }));
+    // finish rows: VERIFY (single-command verifications), a row per verification lane, FINAL REVIEW
+    const finishLanes = [['verification', 'verification', 'VERIFY', 'verify'], ...verify.lanes.map(n => ['vlane:' + n, 'verification', 'VERIFY · ' + n, 'verify · ' + n]),
+      ['review', 'review', 'FINAL REVIEW', 'final review']].filter(([row]) => extra.some(e => e.row === row));
+    finishLanes.forEach(([row], i) => extra.filter(e => e.row === row).forEach(e => { e.lane = 'f' + i; e.x = x(e.start); e.w = Math.max(22, (e.end - e.start) * k); e.y = fy + i * LANE_D + (LANE_D - BLOCK_D) / 2; e.d = BLOCK_D; }));
     const steps = [60, 300, 600, 900, 1800, 3600, 7200, 14400, 43200, 86400];
     const step = steps.find(s => span / s <= 10) || 86400;
     const ruler = [];
     for (let t = Math.ceil(t0 / step) * step; t <= t0 + span; t += step) ruler.push({ t, x: x(t), label: fmtDur(t - t0) });
-    return { items, extra, lanes, finishLanes: finishLanes.map(([k, name], i) => ({ kind: k, name, y: fy + i * LANE_D })), cap, overLimit: lanes > cap, t0, t1: t0 + span, k,
+    return { items, extra, lanes, finishLanes: finishLanes.map(([row, kind, name, label], i) => ({ kind, row, name, label, y: fy + i * LANE_D })), cap, overLimit: lanes > cap, t0, t1: t0 + span, k,
       px: EXEC_PAD * 2 + W, py: fy + finishLanes.length * LANE_D + 10, laneD: LANE_D, ruler, live, nowX: num(now) !== null && live ? x(now) : null, x };
   }
 
@@ -602,8 +654,36 @@
     const final = finalAt(run, last, t);
     const vs = list(run && run.timeline && run.timeline.verifications).filter(v => num(v.started) !== null && v.started <= t);
     const lv = vs.pop();
-    const verification = !lv ? 'PENDING' : num(lv.ended) !== null && lv.ended <= t ? (lv.exit_code === 0 ? 'PASS' : 'FAIL') : 'RUNNING';
-    return { t, tasks, final, verification, integration: integrationStatus(Object.values(tasks).map(s => s.status)) };
+    const verification = !lv ? 'PENDING' : num(lv.ended) !== null && lv.ended <= t ? runStatus(num(lv.exit_code), false) : 'RUNNING';
+    return { t, tasks, final, verification, lanes: lanesAt(run, t), integration: integrationStatus(Object.values(tasks).map(s => s.status)) };
+  }
+  // The finish node's verification LEDs: one per lane coloured by its state (at the replay time when `at` is
+  // stateAt's), else one per run of the command by its exit; a run with no exit is UNKNOWN, never FAIL.
+  const LED_STATE = { pass: 'PASS', fail: 'FAIL', running: 'RUNNING', flaky: 'FLAKY', known: 'KNOWN', interrupted: 'INTERRUPTED' };
+  function verifyLeds(run, at) {
+    const v = run && run.verification;
+    if (v && v.mode === 'lanes' && list(v.lanes).length) {
+      if (at && at.lanes) return at.lanes.map(l => l.status === 'PENDING' ? null : l.status);
+      return list(v.lanes).filter(l => l && l.name).map(l => LED_STATE[l.state] || null);
+    }
+    return list(v && v.runs).map(r => runStatus(num(r.exit), false));
+  }
+  // Each verification lane at time t (lanes mode, else null): PENDING before its first run, RUNNING during a run,
+  // between runs and after the last the status of the run that ended last, a flaky or known lane's own at its end.
+  function lanesAt(run, t) {
+    const v = run && run.verification;
+    if (!v || v.mode !== 'lanes' || !list(v.lanes).length) return null;
+    return list(v.lanes).filter(l => l && l.name).map(l => {
+      let status = 'PENDING';
+      const runs = list(l.runs);
+      runs.forEach((r, i) => {
+        const s = num(r.started), e = num(r.ended), last = i === runs.length - 1;
+        if (s === null || s > t) return;
+        if (e === null || e > t) { status = 'RUNNING'; return; }
+        status = last && LANE_STATUS[l.state] ? LANE_STATUS[l.state] : runStatus(num(r.exit), false);
+      });
+      return { name: l.name, status };
+    });
   }
   function timeRange(run, now) {
     const s = [], e = [];
@@ -618,6 +698,7 @@
     for (const m of list(run && run.integration && run.integration.merged)) if (num(m && m.at) !== null) e.push(m.at);
     for (const r of list(run && run.final_reviews)) { const v = num(r.started) ?? num(r.at); if (v !== null) s.push(v); if (num(r.ended) !== null) e.push(r.ended); }
     for (const v of list(run && run.timeline && run.timeline.verifications)) { if (num(v.started) !== null) s.push(v.started); if (num(v.ended) !== null) e.push(v.ended); }
+    for (const l of list(run && run.verification && run.verification.lanes)) for (const r of list(l && l.runs)) if (num(r && r.ended) !== null) e.push(r.ended);
     if (!s.length) return null;
     return { start: Math.min(...s), end: Math.max(...e, ...s) };
   }
@@ -777,6 +858,6 @@
 
   return { liveNow, runState, execStatus, matchRun, filterOptions, atlasModel, columns, planModel, finishState,
     finalStatus, runReviewStatus, reviewSinceLine, reviewAt, taskStatus, taskVerdict, attemptStatus, taskParts,
-    taskTimes, metricValue, overlap, projectSchedule, execModel, criticalPath, concurrency, stateAt, timeRange, events, lineage, structureKey, innerKey,
+    taskTimes, metricValue, overlap, projectSchedule, execModel, verifyLeds, criticalPath, concurrency, stateAt, timeRange, events, lineage, structureKey, innerKey,
     fmtDur, alias, basename, weight, searchIndex, searchRank, statusEvents, fractalTree };
 });