@swarmclawai/swarmclaw 1.9.5 → 1.9.7

package/src/components/quality/quality-workspace.tsx

@@ -17,7 +17,7 @@ import {
 } from '@/lib/quality/quality-summary'
 import { cn } from '@/lib/utils'
 import { useAppStore } from '@/stores/use-app-store'
-import type { EvalRun, EvalSuiteResult } from '@/lib/server/eval/types'
+import type { EvalEnvironmentPlan, EvalGateResult, EvalRun, EvalSuiteResult } from '@/lib/server/eval/types'
 import type { Agent, ApprovalRequest, SessionRunRecord } from '@/types'
 
 type QualityTab = 'overview' | 'evals' | 'approvals' | 'runs'
@@ -105,6 +105,217 @@ function EmptyState({ title, description }: { title: string; description: string
   )
 }
 
+function environmentStatusClass(status: EvalEnvironmentPlan['status']): string {
+  if (status === 'ready') return 'border-emerald-500/25 bg-emerald-500/10 text-emerald-200'
+  if (status === 'warning') return 'border-amber-500/25 bg-amber-500/10 text-amber-200'
+  return 'border-rose-500/25 bg-rose-500/10 text-rose-200'
+}
+
+function checkClass(level: 'info' | 'warn' | 'error'): string {
+  if (level === 'error') return 'border-rose-500/20 bg-rose-500/[0.05] text-rose-200'
+  if (level === 'warn') return 'border-amber-500/20 bg-amber-500/[0.05] text-amber-200'
+  return 'border-white/[0.06] bg-white/[0.025] text-text-3'
+}
+
+function gateStatusClass(status: EvalGateResult['status']): string {
+  if (status === 'pass') return 'border-emerald-500/25 bg-emerald-500/10 text-emerald-200'
+  if (status === 'warn') return 'border-amber-500/25 bg-amber-500/10 text-amber-200'
+  return 'border-rose-500/25 bg-rose-500/10 text-rose-200'
+}
+
+function gateCheckClass(status: EvalGateResult['status']): string {
+  if (status === 'fail') return 'border-rose-500/20 bg-rose-500/[0.05] text-rose-200'
+  if (status === 'warn') return 'border-amber-500/20 bg-amber-500/[0.05] text-amber-200'
+  return 'border-emerald-500/20 bg-emerald-500/[0.05] text-emerald-200'
+}
+
+function EvalEnvironmentPanel({ plan, loading, onRefresh }: {
+  plan: EvalEnvironmentPlan | null
+  loading: boolean
+  onRefresh: () => void
+}) {
+  return (
+    <div className="rounded-[12px] border border-white/[0.06] bg-white/[0.025] px-3 py-3">
+      <div className="flex items-start justify-between gap-3">
+        <div>
+          <div className="text-[13px] font-800 text-text">Validation environment</div>
+          <p className="mt-1 text-[11px] leading-relaxed text-text-3/65">
+            Preflight checks, workspace context, and generated files for the selected eval.
+          </p>
+        </div>
+        <button
+          type="button"
+          onClick={onRefresh}
+          disabled={loading}
+          className="shrink-0 rounded-[8px] border border-white/[0.08] px-2 py-1 text-[10px] font-800 text-text-2 transition-colors hover:bg-white/[0.06] disabled:opacity-40"
+        >
+          {loading ? 'Checking' : 'Refresh'}
+        </button>
+      </div>
+      {!plan ? (
+        <div className="mt-3 text-[11px] text-text-3/60">{loading ? 'Checking readiness...' : 'Choose an agent and scenario.'}</div>
+      ) : (
+        <div className="mt-3 flex flex-col gap-3">
+          <div className="flex flex-wrap items-center gap-2">
+            <span className={cn('rounded-full border px-2 py-1 text-[10px] font-800 uppercase tracking-[0.08em]', environmentStatusClass(plan.status))}>
+              {plan.status}
+            </span>
+            {plan.target && (
+              <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+                {plan.target.kind} - {plan.target.label}
+              </span>
+            )}
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {plan.requiredTools.length} tool{plan.requiredTools.length === 1 ? '' : 's'}
+            </span>
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {plan.generatedFiles.length} file{plan.generatedFiles.length === 1 ? '' : 's'}
+            </span>
+          </div>
+          {plan.target?.environmentLabel && (
+            <div className="rounded-[10px] border border-white/[0.06] bg-white/[0.02] px-3 py-2 text-[11px] text-text-3/70">
+              Environment: <span className="font-700 text-text-2">{plan.target.environmentLabel}</span>
+              {plan.target.environmentStatus ? ` (${plan.target.environmentStatus})` : ''}
+            </div>
+          )}
+          <div className="flex flex-col gap-1.5">
+            {plan.checks.slice(0, 4).map((check) => (
+              <div key={`${check.code}:${check.message}`} className={cn('rounded-[9px] border px-2.5 py-2 text-[11px] leading-relaxed', checkClass(check.level))}>
+                <span className="font-800 uppercase tracking-[0.08em]">{check.level}</span>
+                <span className="ml-2">{check.message}</span>
+              </div>
+            ))}
+            {plan.checks.length > 4 && (
+              <div className="text-[10px] text-text-3/55">+{plan.checks.length - 4} more check{plan.checks.length - 4 === 1 ? '' : 's'}</div>
+            )}
+          </div>
+          <div className="flex flex-wrap gap-1.5">
+            {plan.generatedFiles.slice(0, 5).map((file) => (
+              <span key={`${file.kind}:${file.path}`} className="rounded-full bg-white/[0.04] px-2 py-1 text-[10px] font-700 text-text-3">
+                {file.path}
+              </span>
+            ))}
+            {plan.generatedFiles.length > 5 && (
+              <span className="rounded-full bg-white/[0.04] px-2 py-1 text-[10px] font-700 text-text-3">
+                +{plan.generatedFiles.length - 5}
+              </span>
+            )}
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+function EvalGatePanel({
+  gate,
+  loading,
+  busy,
+  scope,
+  onScopeChange,
+  onRefresh,
+  onSetBaseline,
+}: {
+  gate: EvalGateResult | null
+  loading: boolean
+  busy: boolean
+  scope: 'scenario' | 'suite'
+  onScopeChange: (scope: 'scenario' | 'suite') => void
+  onRefresh: () => void
+  onSetBaseline: () => void
+}) {
+  return (
+    <div className="rounded-[12px] border border-white/[0.06] bg-white/[0.025] px-3 py-3">
+      <div className="flex items-start justify-between gap-3">
+        <div>
+          <div className="text-[13px] font-800 text-text">Regression gate</div>
+          <p className="mt-1 text-[11px] leading-relaxed text-text-3/65">
+            Compare latest eval evidence against thresholds and an approved baseline.
+          </p>
+        </div>
+        <button
+          type="button"
+          onClick={onRefresh}
+          disabled={loading}
+          className="shrink-0 rounded-[8px] border border-white/[0.08] px-2 py-1 text-[10px] font-800 text-text-2 transition-colors hover:bg-white/[0.06] disabled:opacity-40"
+        >
+          {loading ? 'Checking' : 'Refresh'}
+        </button>
+      </div>
+
+      <div className="mt-3 flex rounded-[10px] border border-white/[0.06] bg-white/[0.025] p-1">
+        {(['scenario', 'suite'] as const).map((item) => (
+          <button
+            key={item}
+            type="button"
+            onClick={() => onScopeChange(item)}
+            className={cn(
+              'flex-1 rounded-[8px] px-2 py-1.5 text-[10px] font-800 uppercase tracking-[0.08em] transition-colors',
+              scope === item ? 'bg-white/[0.1] text-text' : 'text-text-3 hover:bg-white/[0.05]',
+            )}
+          >
+            {item}
+          </button>
+        ))}
+      </div>
+
+      {!gate ? (
+        <div className="mt-3 text-[11px] text-text-3/60">{loading ? 'Checking gate...' : 'Run evals to build gate evidence.'}</div>
+      ) : (
+        <div className="mt-3 flex flex-col gap-3">
+          <div className="flex flex-wrap items-center gap-2">
+            <span className={cn('rounded-full border px-2 py-1 text-[10px] font-800 uppercase tracking-[0.08em]', gateStatusClass(gate.status))}>
+              {gate.status}
+            </span>
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {gate.scope.label}
+            </span>
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {gate.latestRuns.length}/{gate.scope.scenarioIds.length} latest runs
+            </span>
+          </div>
+
+          <div className="grid grid-cols-3 gap-2">
+            <div className="rounded-[10px] border border-white/[0.06] bg-white/[0.02] px-2 py-2">
+              <div className="text-[9px] font-800 uppercase tracking-[0.08em] text-text-3/50">Current</div>
+              <div className="mt-1 text-[14px] font-800 text-text">{formatPercent(gate.currentPercent)}</div>
+            </div>
+            <div className="rounded-[10px] border border-white/[0.06] bg-white/[0.02] px-2 py-2">
+              <div className="text-[9px] font-800 uppercase tracking-[0.08em] text-text-3/50">Baseline</div>
+              <div className="mt-1 text-[14px] font-800 text-text">{gate.baseline ? `${gate.baseline.baselinePercent}%` : 'none'}</div>
+            </div>
+            <div className="rounded-[10px] border border-white/[0.06] bg-white/[0.02] px-2 py-2">
+              <div className="text-[9px] font-800 uppercase tracking-[0.08em] text-text-3/50">Regression</div>
+              <div className="mt-1 text-[14px] font-800 text-text">{gate.regressionPoints == null ? 'n/a' : `${gate.regressionPoints}pt`}</div>
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-1.5">
+            {gate.checks.slice(0, 4).map((check) => (
+              <div key={`${check.code}:${check.message}`} className={cn('rounded-[9px] border px-2.5 py-2 text-[11px] leading-relaxed', gateCheckClass(check.status))}>
+                <span className="font-800 uppercase tracking-[0.08em]">{check.status}</span>
+                <span className="ml-2">{check.message}</span>
+              </div>
+            ))}
+            {gate.checks.length > 4 && (
+              <div className="text-[10px] text-text-3/55">+{gate.checks.length - 4} more check{gate.checks.length - 4 === 1 ? '' : 's'}</div>
+            )}
+          </div>
+
+          <button
+            type="button"
+            onClick={onSetBaseline}
+            disabled={busy || gate.latestRuns.length === 0 || gate.checks.some((check) => check.code === 'missing_scope_runs')}
+            className="rounded-[9px] border border-white/[0.08] bg-white/[0.04] px-3 py-2 text-[11px] font-800 text-text-2 transition-colors hover:bg-white/[0.08] disabled:cursor-not-allowed disabled:opacity-40"
+          >
+            {busy ? 'Saving baseline' : gate.baseline ? 'Update baseline' : 'Set baseline'}
+          </button>
+        </div>
+      )}
+    </div>
+  )
+}
+
 export function QualityWorkspace() {
   const router = useRouter()
   const searchParams = useSearchParams()
@@ -127,6 +338,12 @@ export function QualityWorkspace() {
   const [selectedSuite, setSelectedSuite] = useState('core')
   const [selectedScenarioId, setSelectedScenarioId] = useState('')
   const [evalBusy, setEvalBusy] = useState<string | null>(null)
+  const [evalEnvironmentPlan, setEvalEnvironmentPlan] = useState<EvalEnvironmentPlan | null>(null)
+  const [evalEnvironmentLoading, setEvalEnvironmentLoading] = useState(false)
+  const [evalGate, setEvalGate] = useState<EvalGateResult | null>(null)
+  const [evalGateScope, setEvalGateScope] = useState<'scenario' | 'suite'>('scenario')
+  const [evalGateLoading, setEvalGateLoading] = useState(false)
+  const [evalBaselineBusy, setEvalBaselineBusy] = useState(false)
   const [approvalBusy, setApprovalBusy] = useState<string | null>(null)
 
   useEffect(() => {
@@ -170,6 +387,51 @@ export function QualityWorkspace() {
     }
   }, [])
 
+  const loadEvalEnvironmentPlan = useCallback(async (opts: { refreshGateway?: boolean } = {}) => {
+    if (!selectedAgentId) {
+      setEvalEnvironmentPlan(null)
+      return
+    }
+    const params = new URLSearchParams({ agentId: selectedAgentId })
+    if (selectedScenarioId) params.set('scenarioId', selectedScenarioId)
+    else if (selectedSuite) params.set('suite', selectedSuite)
+    if (opts.refreshGateway) params.set('refreshGateway', 'true')
+    setEvalEnvironmentLoading(true)
+    try {
+      const plan = await api<EvalEnvironmentPlan>('GET', `/eval/environments?${params.toString()}`, undefined, { timeoutMs: opts.refreshGateway ? 20_000 : 8_000 })
+      setEvalEnvironmentPlan(plan)
+    } catch (err) {
+      setEvalEnvironmentPlan(null)
+      toast.error(err instanceof Error ? err.message : 'Unable to validate eval environment')
+    } finally {
+      setEvalEnvironmentLoading(false)
+    }
+  }, [selectedAgentId, selectedScenarioId, selectedSuite])
+
+  const loadEvalGate = useCallback(async () => {
+    if (!selectedAgentId) {
+      setEvalGate(null)
+      return
+    }
+    if (evalGateScope === 'scenario' && !selectedScenarioId) {
+      setEvalGate(null)
+      return
+    }
+    const params = new URLSearchParams({ agentId: selectedAgentId })
+    if (evalGateScope === 'scenario') params.set('scenarioId', selectedScenarioId)
+    else params.set('suite', selectedSuite)
+    setEvalGateLoading(true)
+    try {
+      const gate = await api<EvalGateResult>('GET', `/eval/gate?${params.toString()}`)
+      setEvalGate(gate)
+    } catch (err) {
+      setEvalGate(null)
+      toast.error(err instanceof Error ? err.message : 'Unable to check eval gate')
+    } finally {
+      setEvalGateLoading(false)
+    }
+  }, [evalGateScope, selectedAgentId, selectedScenarioId, selectedSuite])
+
   useEffect(() => {
     void loadQualityData()
   }, [loadQualityData])
@@ -184,6 +446,14 @@ export function QualityWorkspace() {
     if (!selectedScenarioId && scenarios[0]) setSelectedScenarioId(scenarios[0].id)
   }, [scenarios, selectedScenarioId])
 
+  useEffect(() => {
+    void loadEvalEnvironmentPlan()
+  }, [loadEvalEnvironmentPlan])
+
+  useEffect(() => {
+    void loadEvalGate()
+  }, [loadEvalGate])
+
   useEffect(() => {
     if (!suites.some((suite) => suite.name === selectedSuite) && suites[0]) {
       setSelectedSuite(suites[0].name)
@@ -208,34 +478,82 @@ export function QualityWorkspace() {
       toast.error('Choose an agent and scenario first')
       return
     }
+    if (evalEnvironmentPlan?.status === 'blocked') {
+      toast.error('Fix the validation environment before running this eval')
+      return
+    }
     setEvalBusy(`scenario:${selectedScenarioId}`)
     try {
-      await api<EvalRun>('POST', '/eval/run', { agentId: selectedAgentId, scenarioId: selectedScenarioId }, { timeoutMs: 180_000 })
+      await api<EvalRun>('POST', '/eval/run', {
+        agentId: selectedAgentId,
+        scenarioId: selectedScenarioId,
+        gatewayProfileId: evalEnvironmentPlan?.target?.gatewayProfileId || null,
+        environmentId: evalEnvironmentPlan?.target?.environmentId || null,
+        refreshGateway: evalEnvironmentPlan?.target?.kind === 'gateway',
+      }, { timeoutMs: 180_000 })
       toast.success('Eval scenario completed')
       await loadQualityData({ silent: true })
+      await loadEvalEnvironmentPlan()
+      await loadEvalGate()
     } catch (err) {
       toast.error(err instanceof Error ? err.message : 'Eval scenario failed')
     } finally {
       setEvalBusy(null)
     }
-  }, [loadQualityData, selectedAgentId, selectedScenarioId])
+  }, [evalEnvironmentPlan, loadEvalEnvironmentPlan, loadEvalGate, loadQualityData, selectedAgentId, selectedScenarioId])
 
   const runSuite = useCallback(async (suiteName: string) => {
     if (!selectedAgentId) {
       toast.error('Choose an agent first')
       return
     }
+    if (evalEnvironmentPlan?.status === 'blocked') {
+      toast.error('Fix the validation environment before running this suite')
+      return
+    }
     setEvalBusy(`suite:${suiteName}`)
     try {
-      const result = await api<EvalSuiteResult>('POST', '/eval/suite', { agentId: selectedAgentId, suite: suiteName }, { timeoutMs: 300_000 })
+      const result = await api<EvalSuiteResult>('POST', '/eval/suite', {
+        agentId: selectedAgentId,
+        suite: suiteName,
+        gatewayProfileId: evalEnvironmentPlan?.target?.gatewayProfileId || null,
+        environmentId: evalEnvironmentPlan?.target?.environmentId || null,
+        refreshGateway: evalEnvironmentPlan?.target?.kind === 'gateway',
+      }, { timeoutMs: 300_000 })
       toast.success(`Suite completed at ${Math.round(result.percentage)}%`)
       await loadQualityData({ silent: true })
+      await loadEvalEnvironmentPlan()
+      await loadEvalGate()
     } catch (err) {
       toast.error(err instanceof Error ? err.message : 'Eval suite failed')
     } finally {
       setEvalBusy(null)
     }
-  }, [loadQualityData, selectedAgentId])
+  }, [evalEnvironmentPlan, loadEvalEnvironmentPlan, loadEvalGate, loadQualityData, selectedAgentId])
+
+  const setEvalBaseline = useCallback(async () => {
+    if (!selectedAgentId) {
+      toast.error('Choose an agent first')
+      return
+    }
+    if (evalGateScope === 'scenario' && !selectedScenarioId) {
+      toast.error('Choose a scenario first')
+      return
+    }
+    setEvalBaselineBusy(true)
+    try {
+      const body = evalGateScope === 'scenario'
+        ? { agentId: selectedAgentId, scenarioId: selectedScenarioId, minPercent: evalGate?.minPercent ?? 80, maxRegressionPoints: evalGate?.maxRegressionPoints ?? 5 }
+        : { agentId: selectedAgentId, suite: selectedSuite, minPercent: evalGate?.minPercent ?? 80, maxRegressionPoints: evalGate?.maxRegressionPoints ?? 5 }
+      const result = await api<{ gate: EvalGateResult }>('POST', '/eval/baselines', body)
+      setEvalGate(result.gate)
+      toast.success('Eval baseline saved')
+    } catch (err) {
+      toast.error(err instanceof Error ? err.message : 'Unable to save eval baseline')
+    } finally {
+      setEvalBaselineBusy(false)
+    }
+  }, [evalGate, evalGateScope, selectedAgentId, selectedScenarioId, selectedSuite])
 
   const actOnApproval = useCallback(async (approval: ApprovalRequest, approved: boolean) => {
     setApprovalBusy(approval.id)
@@ -456,6 +774,20 @@ export function QualityWorkspace() {
                       </div>
                     </div>
                   )}
+                  <EvalEnvironmentPanel
+                    plan={evalEnvironmentPlan}
+                    loading={evalEnvironmentLoading}
+                    onRefresh={() => void loadEvalEnvironmentPlan({ refreshGateway: true })}
+                  />
+                  <EvalGatePanel
+                    gate={evalGate}
+                    loading={evalGateLoading}
+                    busy={evalBaselineBusy}
+                    scope={evalGateScope}
+                    onScopeChange={setEvalGateScope}
+                    onRefresh={() => void loadEvalGate()}
+                    onSetBaseline={() => void setEvalBaseline()}
+                  />
                   <button
                     type="button"
                     onClick={() => openMissionTemplate('release-candidate-qa')}

package/src/lib/server/eval/baseline.test.ts

@@ -0,0 +1,111 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+
+import {
+  evaluateEvalGate,
+  setEvalBaseline,
+} from './baseline'
+import type { EvalBaseline, EvalRun } from './types'
+
+function makeRun(overrides: Partial<EvalRun> = {}): EvalRun {
+  return {
+    id: 'run-1',
+    scenarioId: 'coding-prime',
+    agentId: 'agent-1',
+    status: 'completed',
+    startedAt: 1,
+    endedAt: 2,
+    score: 8,
+    maxScore: 10,
+    details: [],
+    ...overrides,
+  }
+}
+
+function depsFor(runs: EvalRun[], baseline: EvalBaseline | null = null, saved: EvalBaseline[] = []) {
+  return {
+    now: () => 123,
+    listRunsByAgent: (agentId: string) => runs.filter((run) => run.agentId === agentId),
+    getBaselineForScope: () => baseline,
+    saveBaseline: (next: EvalBaseline) => { saved.push(next) },
+  }
+}
+
+test('setEvalBaseline snapshots the latest scenario score and gate defaults', () => {
+  const saved: EvalBaseline[] = []
+  const baseline = setEvalBaseline(
+    {
+      agentId: 'agent-1',
+      scenarioId: 'coding-prime',
+      minPercent: 75,
+      maxRegressionPoints: 3,
+      label: 'Release candidate',
+    },
+    depsFor([
+      makeRun({ id: 'older', score: 4, startedAt: 1, endedAt: 2 }),
+      makeRun({ id: 'latest', score: 8, startedAt: 5, endedAt: 6 }),
+    ], null, saved),
+  )
+
+  assert.equal(saved.length, 1)
+  assert.equal(baseline.scope.type, 'scenario')
+  assert.equal(baseline.scope.id, 'coding-prime')
+  assert.equal(baseline.baselinePercent, 80)
+  assert.equal(baseline.minPercent, 75)
+  assert.equal(baseline.maxRegressionPoints, 3)
+  assert.deepEqual(baseline.runIds, ['latest'])
+})
+
+test('evaluateEvalGate warns until a baseline is approved', () => {
+  const gate = evaluateEvalGate(
+    { agentId: 'agent-1', scenarioId: 'coding-prime', minPercent: 70 },
+    depsFor([makeRun({ score: 8, maxScore: 10 })]),
+  )
+
+  assert.equal(gate.currentPercent, 80)
+  assert.equal(gate.status, 'warn')
+  assert.ok(gate.checks.some((check) => check.code === 'baseline_missing' && check.status === 'warn'))
+})
+
+test('evaluateEvalGate fails when regression exceeds the baseline allowance', () => {
+  const baseline = setEvalBaseline(
+    { agentId: 'agent-1', scenarioId: 'coding-prime', minPercent: 70, maxRegressionPoints: 2 },
+    depsFor([makeRun({ id: 'baseline', score: 9, maxScore: 10 })]),
+  )
+
+  const gate = evaluateEvalGate(
+    { agentId: 'agent-1', scenarioId: 'coding-prime' },
+    depsFor([makeRun({ id: 'current', score: 6, maxScore: 10, startedAt: 10, endedAt: 11 })], baseline),
+  )
+
+  assert.equal(gate.currentPercent, 60)
+  assert.equal(gate.regressionPoints, 30)
+  assert.equal(gate.status, 'fail')
+  assert.ok(gate.checks.some((check) => check.code === 'regression_limit_exceeded'))
+})
+
+test('evaluateEvalGate passes when score and regression checks pass', () => {
+  const baseline = setEvalBaseline(
+    { agentId: 'agent-1', scenarioId: 'coding-prime', minPercent: 70, maxRegressionPoints: 5 },
+    depsFor([makeRun({ id: 'baseline', score: 8, maxScore: 10 })]),
+  )
+
+  const gate = evaluateEvalGate(
+    { agentId: 'agent-1', scenarioId: 'coding-prime' },
+    depsFor([makeRun({ id: 'current', score: 8, maxScore: 10, startedAt: 10, endedAt: 11 })], baseline),
+  )
+
+  assert.equal(gate.status, 'pass')
+  assert.equal(gate.regressionPoints, 0)
+  assert.ok(gate.checks.some((check) => check.code === 'score_threshold_met'))
+})
+
+test('suite gates require latest runs for every scenario in scope before baselining', () => {
+  assert.throws(
+    () => setEvalBaseline(
+      { agentId: 'agent-1', suite: 'core' },
+      depsFor([makeRun({ scenarioId: 'coding-prime' })]),
+    ),
+    /Baseline requires latest runs for every scenario in scope/,
+  )
+})