@swarmclawai/swarmclaw 1.9.5 → 1.9.6

package/README.md

@@ -399,6 +399,16 @@ Operational docs: https://swarmclaw.ai/docs/observability
 
 ## Releases
 
+### v1.9.6 Highlights
+
+Bundled eval-environment release: validation preflights, deterministic eval workspaces, and clearer operator readiness before spending run budget.
+
+- **Eval validation environments.** `/api/eval/environments` now resolves the selected agent route, gateway target, scenario tools, generated files, and readiness checks before an eval runs.
+- **Workspace manifests.** Eval runs now write `environment.json`, `.env.swarmclaw-eval`, and a task-focused `README.md` into each isolated eval workspace without embedding secrets.
+- **Scenario fixtures.** Eval scenarios can declare fixture files, and the package-analysis scenario now gets a deterministic `package.json` in its workspace.
+- **Fail-fast readiness.** Blocked evals stop before model execution when the agent route, CLI provider, gateway profile, or execution environment is not ready.
+- **Quality UI preflight.** The Eval Lab now shows target status, gateway environment, checks, tools, and generated files next to the selected scenario.
+
 ### v1.9.5 Highlights
 
 Bundled portability release: project-scoped workspace bundles, safer v2 imports, and preserved internal relationships for reusable teams.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swarmclawai/swarmclaw",
-  "version": "1.9.5",
+  "version": "1.9.6",
   "description": "Build and run autonomous AI agents with OpenClaw, Hermes, multiple model providers, orchestration, delegation, memory, skills, schedules, and chat connectors.",
   "main": "electron-dist/main.js",
   "license": "MIT",
@@ -87,7 +87,7 @@
     "test:cli": "node --test src/cli/*.test.js bin/*.test.js scripts/electron-after-pack.test.mjs scripts/ensure-sandbox-browser-image.test.mjs scripts/postinstall.test.mjs scripts/run-next-build.test.mjs scripts/run-next-typegen.test.mjs",
     "test:setup": "tsx --test src/app/api/setup/check-provider/route.test.ts src/lib/server/provider-model-discovery.test.ts src/components/auth/setup-wizard/utils.test.ts src/components/auth/setup-wizard/types.test.ts src/hooks/setup-done-detection.test.ts src/lib/setup-defaults.test.ts src/lib/server/storage-auth.test.ts src/lib/server/storage-auth-docker.test.ts",
     "test:openclaw": "tsx --test src/lib/openclaw/openclaw-agent-id.test.ts src/lib/openclaw/openclaw-endpoint.test.ts src/lib/server/agents/agent-runtime-config.test.ts src/lib/server/build-llm.test.ts src/lib/server/connectors/connector-routing.test.ts src/lib/server/connectors/openclaw.test.ts src/lib/server/connectors/swarmdock.test.ts src/lib/server/gateway/protocol.test.ts src/lib/server/gateways/gateway-topology.test.ts src/lib/server/llm-response-cache.test.ts src/lib/server/mcp-conformance.test.ts src/lib/server/openclaw/agent-resolver.test.ts src/lib/server/openclaw/deploy.test.ts src/lib/server/openclaw/skills-normalize.test.ts src/lib/server/session-tools/openclaw-nodes.test.ts src/lib/server/session-tools/swarmdock.test.ts src/lib/server/tasks/task-quality-gate.test.ts src/lib/server/tasks/task-validation.test.ts src/lib/server/tool-capability-policy.test.ts src/lib/providers/openai.test.ts src/lib/providers/openclaw-exports.test.ts src/app/api/gateways/topology-route.test.ts src/app/api/openclaw/dashboard-url/route.test.ts",
-    "test:runtime": "tsx --test src/lib/a2a/agent-card.test.ts src/lib/strip-internal-metadata.test.ts src/lib/provider-sets.test.ts src/lib/providers/opencode-cli.test.ts src/lib/providers/cli-provider-metadata.test.ts src/lib/providers/cli-utils.test.ts src/lib/providers/generic-cli.test.ts src/lib/server/agents/delegation-advisory.test.ts src/lib/server/cli-provider-readiness.test.ts src/lib/server/provider-health.test.ts src/lib/server/mcp-gateway-runtime.test.ts src/lib/server/mcp-connection-pool.test.ts src/lib/server/knowledge-sources.test.ts src/lib/server/extension-managed-resources.test.ts src/lib/server/chat-execution/chat-execution-grounding.test.ts src/lib/server/chat-execution/chat-turn-preparation.test.ts src/lib/server/chat-execution/iteration-timers.test.ts src/lib/server/chat-execution/post-stream-finalization.test.ts src/lib/server/chat-execution/reasoning-tag-scrubber.test.ts src/lib/server/chats/clear-undo-snapshots.test.ts src/lib/server/connectors/email.test.ts src/lib/server/protocols/protocol-service.test.ts src/lib/server/runtime/run-ledger.test.ts src/lib/server/runtime/queue-retry-policy.test.ts src/lib/server/runs/run-brief.test.ts src/lib/server/operations/operation-pulse.test.ts src/lib/server/artifacts/artifact-resolver.test.ts src/lib/server/observability/otel-config.test.ts src/lib/server/safe-parse-body.test.ts src/lib/server/missions/mission-templates.test.ts src/lib/server/sharing/share-link-repository.test.ts src/lib/server/sharing/share-resolver.test.ts src/lib/server/tasks/task-execution-workspace.test.ts src/lib/server/tasks/task-service.test.ts src/lib/server/session-tools/execute.test.ts src/lib/server/session-tools/manage-tasks.test.ts src/lib/app/view-constants.test.ts src/lib/quality/quality-summary.test.ts src/app/api/approvals/route.test.ts src/app/api/agents/agents-route.test.ts src/app/api/tasks/tasks-route.test.ts src/app/api/tasks/task-workspace-route.test.ts src/app/api/chats/chat-route.test.ts src/app/api/chats/clear-route.test.ts src/app/api/chats/compact-route.test.ts src/app/api/chats/context-status-route.test.ts src/app/api/connectors/connector-doctor-route.test.ts src/app/api/extensions/managed-resources/route.test.ts src/app/api/healthz/route.test.ts src/app/api/logs/route.test.ts src/app/api/portability/export/route.test.ts src/app/api/portability/import/route.test.ts src/app/api/providers/[id]/route.test.ts src/app/api/tts/route.test.ts",
+    "test:runtime": "tsx --test src/lib/a2a/agent-card.test.ts src/lib/strip-internal-metadata.test.ts src/lib/provider-sets.test.ts src/lib/providers/opencode-cli.test.ts src/lib/providers/cli-provider-metadata.test.ts src/lib/providers/cli-utils.test.ts src/lib/providers/generic-cli.test.ts src/lib/server/agents/delegation-advisory.test.ts src/lib/server/cli-provider-readiness.test.ts src/lib/server/provider-health.test.ts src/lib/server/mcp-gateway-runtime.test.ts src/lib/server/mcp-connection-pool.test.ts src/lib/server/knowledge-sources.test.ts src/lib/server/extension-managed-resources.test.ts src/lib/server/eval/environment-plan.test.ts src/lib/server/chat-execution/chat-execution-grounding.test.ts src/lib/server/chat-execution/chat-turn-preparation.test.ts src/lib/server/chat-execution/iteration-timers.test.ts src/lib/server/chat-execution/post-stream-finalization.test.ts src/lib/server/chat-execution/reasoning-tag-scrubber.test.ts src/lib/server/chats/clear-undo-snapshots.test.ts src/lib/server/connectors/email.test.ts src/lib/server/protocols/protocol-service.test.ts src/lib/server/runtime/run-ledger.test.ts src/lib/server/runtime/queue-retry-policy.test.ts src/lib/server/runs/run-brief.test.ts src/lib/server/operations/operation-pulse.test.ts src/lib/server/artifacts/artifact-resolver.test.ts src/lib/server/observability/otel-config.test.ts src/lib/server/safe-parse-body.test.ts src/lib/server/missions/mission-templates.test.ts src/lib/server/sharing/share-link-repository.test.ts src/lib/server/sharing/share-resolver.test.ts src/lib/server/tasks/task-execution-workspace.test.ts src/lib/server/tasks/task-service.test.ts src/lib/server/session-tools/execute.test.ts src/lib/server/session-tools/manage-tasks.test.ts src/lib/app/view-constants.test.ts src/lib/quality/quality-summary.test.ts src/app/api/approvals/route.test.ts src/app/api/agents/agents-route.test.ts src/app/api/tasks/tasks-route.test.ts src/app/api/tasks/task-workspace-route.test.ts src/app/api/chats/chat-route.test.ts src/app/api/chats/clear-route.test.ts src/app/api/chats/compact-route.test.ts src/app/api/chats/context-status-route.test.ts src/app/api/connectors/connector-doctor-route.test.ts src/app/api/extensions/managed-resources/route.test.ts src/app/api/healthz/route.test.ts src/app/api/logs/route.test.ts src/app/api/portability/export/route.test.ts src/app/api/portability/import/route.test.ts src/app/api/providers/[id]/route.test.ts src/app/api/tts/route.test.ts",
     "test:builder": "tsx --test src/features/protocols/builder/utils/nodes-to-template.test.ts src/features/protocols/builder/utils/template-to-nodes.test.ts src/features/protocols/builder/validators/dag-validator.test.ts",
     "test:e2e": "node --import tsx scripts/browser-e2e-smoke.ts",
     "test:mcp:conformance": "node --import tsx ./scripts/mcp-conformance-check.ts",

package/src/app/api/eval/environments/route.ts

@@ -0,0 +1,59 @@
+import { NextResponse } from 'next/server'
+import { z } from 'zod'
+
+import { buildEvalEnvironmentPlan } from '@/lib/server/eval/environment-plan'
+import { errorMessage } from '@/lib/shared-utils'
+
+const PlanSchema = z.object({
+  agentId: z.string().min(1),
+  scenarioId: z.string().min(1).nullable().optional(),
+  suite: z.string().min(1).nullable().optional(),
+  gatewayProfileId: z.string().min(1).nullable().optional(),
+  environmentId: z.string().min(1).nullable().optional(),
+  refreshGateway: z.boolean().optional(),
+})
+
+function readBoolean(value: string | null): boolean {
+  return value === '1' || value === 'true'
+}
+
+export async function GET(req: Request) {
+  try {
+    const { searchParams } = new URL(req.url)
+    const parsed = PlanSchema.safeParse({
+      agentId: searchParams.get('agentId') || '',
+      scenarioId: searchParams.get('scenarioId'),
+      suite: searchParams.get('suite'),
+      gatewayProfileId: searchParams.get('gatewayProfileId'),
+      environmentId: searchParams.get('environmentId'),
+      refreshGateway: readBoolean(searchParams.get('refreshGateway')),
+    })
+    if (!parsed.success) {
+      return NextResponse.json(
+        { error: parsed.error.issues.map((issue) => issue.message).join(', ') },
+        { status: 400 },
+      )
+    }
+    const plan = await buildEvalEnvironmentPlan(parsed.data)
+    return NextResponse.json(plan)
+  } catch (err: unknown) {
+    return NextResponse.json({ error: errorMessage(err) }, { status: 500 })
+  }
+}
+
+export async function POST(req: Request) {
+  try {
+    const body: unknown = await req.json()
+    const parsed = PlanSchema.safeParse(body)
+    if (!parsed.success) {
+      return NextResponse.json(
+        { error: parsed.error.issues.map((issue) => issue.message).join(', ') },
+        { status: 400 },
+      )
+    }
+    const plan = await buildEvalEnvironmentPlan(parsed.data)
+    return NextResponse.json(plan)
+  } catch (err: unknown) {
+    return NextResponse.json({ error: errorMessage(err) }, { status: 500 })
+  }
+}

package/src/app/api/eval/run/route.ts

@@ -7,6 +7,9 @@ import { errorMessage } from '@/lib/shared-utils'
 const RunSchema = z.object({
   scenarioId: z.string().min(1),
   agentId: z.string().min(1),
+  gatewayProfileId: z.string().min(1).nullable().optional(),
+  environmentId: z.string().min(1).nullable().optional(),
+  refreshGateway: z.boolean().optional(),
 })
 
 export async function POST(req: Request) {
@@ -20,7 +23,11 @@ export async function POST(req: Request) {
       )
     }
 
-    const result = await runEvalScenario(parsed.data.scenarioId, parsed.data.agentId)
+    const result = await runEvalScenario(parsed.data.scenarioId, parsed.data.agentId, {
+      gatewayProfileId: parsed.data.gatewayProfileId || null,
+      environmentId: parsed.data.environmentId || null,
+      refreshGateway: parsed.data.refreshGateway === true,
+    })
     return NextResponse.json(result)
   } catch (err: unknown) {
     return NextResponse.json(

package/src/app/api/eval/suite/route.ts

@@ -7,6 +7,9 @@ const SuiteSchema = z.object({
   agentId: z.string().min(1),
   categories: z.array(z.string()).optional(),
   suite: z.string().min(1).optional(),
+  gatewayProfileId: z.string().min(1).nullable().optional(),
+  environmentId: z.string().min(1).nullable().optional(),
+  refreshGateway: z.boolean().optional(),
 })
 
 export async function POST(req: Request) {
@@ -23,6 +26,9 @@ export async function POST(req: Request) {
     const result = await runEvalSuite(parsed.data.agentId, {
       categories: parsed.data.categories,
       suite: parsed.data.suite,
+      gatewayProfileId: parsed.data.gatewayProfileId || null,
+      environmentId: parsed.data.environmentId || null,
+      refreshGateway: parsed.data.refreshGateway === true,
     })
     return NextResponse.json(result)
   } catch (err: unknown) {

package/src/cli/index.js

@@ -231,8 +231,10 @@ const COMMAND_GROUPS = [
       cmd('scenarios', 'GET', '/eval/scenarios', 'List available eval scenarios'),
       cmd('suites', 'GET', '/eval/suites', 'List available eval suites (core, swe-bench-lite, gaia-l1, ...)'),
       cmd('status', 'GET', '/eval/run', 'Get eval run status'),
+      cmd('environment', 'GET', '/eval/environments', 'Preview validation environment readiness for an eval'),
       cmd('run', 'POST', '/eval/run', 'Run an eval scenario against an agent', { expectsJsonBody: true }),
       cmd('suite', 'POST', '/eval/suite', 'Run a full eval suite against an agent (pass { suite: "swe-bench-lite" } in body)', { expectsJsonBody: true }),
+      cmd('environment-prepare', 'POST', '/eval/environments', 'Prepare validation environment readiness for an eval', { expectsJsonBody: true }),
     ],
   },
   {

package/src/components/quality/quality-workspace.tsx

@@ -17,7 +17,7 @@ import {
 } from '@/lib/quality/quality-summary'
 import { cn } from '@/lib/utils'
 import { useAppStore } from '@/stores/use-app-store'
-import type { EvalRun, EvalSuiteResult } from '@/lib/server/eval/types'
+import type { EvalEnvironmentPlan, EvalRun, EvalSuiteResult } from '@/lib/server/eval/types'
 import type { Agent, ApprovalRequest, SessionRunRecord } from '@/types'
 
 type QualityTab = 'overview' | 'evals' | 'approvals' | 'runs'
@@ -105,6 +105,96 @@ function EmptyState({ title, description }: { title: string; description: string
   )
 }
 
+function environmentStatusClass(status: EvalEnvironmentPlan['status']): string {
+  if (status === 'ready') return 'border-emerald-500/25 bg-emerald-500/10 text-emerald-200'
+  if (status === 'warning') return 'border-amber-500/25 bg-amber-500/10 text-amber-200'
+  return 'border-rose-500/25 bg-rose-500/10 text-rose-200'
+}
+
+function checkClass(level: 'info' | 'warn' | 'error'): string {
+  if (level === 'error') return 'border-rose-500/20 bg-rose-500/[0.05] text-rose-200'
+  if (level === 'warn') return 'border-amber-500/20 bg-amber-500/[0.05] text-amber-200'
+  return 'border-white/[0.06] bg-white/[0.025] text-text-3'
+}
+
+function EvalEnvironmentPanel({ plan, loading, onRefresh }: {
+  plan: EvalEnvironmentPlan | null
+  loading: boolean
+  onRefresh: () => void
+}) {
+  return (
+    <div className="rounded-[12px] border border-white/[0.06] bg-white/[0.025] px-3 py-3">
+      <div className="flex items-start justify-between gap-3">
+        <div>
+          <div className="text-[13px] font-800 text-text">Validation environment</div>
+          <p className="mt-1 text-[11px] leading-relaxed text-text-3/65">
+            Preflight checks, workspace context, and generated files for the selected eval.
+          </p>
+        </div>
+        <button
+          type="button"
+          onClick={onRefresh}
+          disabled={loading}
+          className="shrink-0 rounded-[8px] border border-white/[0.08] px-2 py-1 text-[10px] font-800 text-text-2 transition-colors hover:bg-white/[0.06] disabled:opacity-40"
+        >
+          {loading ? 'Checking' : 'Refresh'}
+        </button>
+      </div>
+      {!plan ? (
+        <div className="mt-3 text-[11px] text-text-3/60">{loading ? 'Checking readiness...' : 'Choose an agent and scenario.'}</div>
+      ) : (
+        <div className="mt-3 flex flex-col gap-3">
+          <div className="flex flex-wrap items-center gap-2">
+            <span className={cn('rounded-full border px-2 py-1 text-[10px] font-800 uppercase tracking-[0.08em]', environmentStatusClass(plan.status))}>
+              {plan.status}
+            </span>
+            {plan.target && (
+              <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+                {plan.target.kind} - {plan.target.label}
+              </span>
+            )}
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {plan.requiredTools.length} tool{plan.requiredTools.length === 1 ? '' : 's'}
+            </span>
+            <span className="rounded-full bg-white/[0.05] px-2 py-1 text-[10px] font-700 text-text-3">
+              {plan.generatedFiles.length} file{plan.generatedFiles.length === 1 ? '' : 's'}
+            </span>
+          </div>
+          {plan.target?.environmentLabel && (
+            <div className="rounded-[10px] border border-white/[0.06] bg-white/[0.02] px-3 py-2 text-[11px] text-text-3/70">
+              Environment: <span className="font-700 text-text-2">{plan.target.environmentLabel}</span>
+              {plan.target.environmentStatus ? ` (${plan.target.environmentStatus})` : ''}
+            </div>
+          )}
+          <div className="flex flex-col gap-1.5">
+            {plan.checks.slice(0, 4).map((check) => (
+              <div key={`${check.code}:${check.message}`} className={cn('rounded-[9px] border px-2.5 py-2 text-[11px] leading-relaxed', checkClass(check.level))}>
+                <span className="font-800 uppercase tracking-[0.08em]">{check.level}</span>
+                <span className="ml-2">{check.message}</span>
+              </div>
+            ))}
+            {plan.checks.length > 4 && (
+              <div className="text-[10px] text-text-3/55">+{plan.checks.length - 4} more check{plan.checks.length - 4 === 1 ? '' : 's'}</div>
+            )}
+          </div>
+          <div className="flex flex-wrap gap-1.5">
+            {plan.generatedFiles.slice(0, 5).map((file) => (
+              <span key={`${file.kind}:${file.path}`} className="rounded-full bg-white/[0.04] px-2 py-1 text-[10px] font-700 text-text-3">
+                {file.path}
+              </span>
+            ))}
+            {plan.generatedFiles.length > 5 && (
+              <span className="rounded-full bg-white/[0.04] px-2 py-1 text-[10px] font-700 text-text-3">
+                +{plan.generatedFiles.length - 5}
+              </span>
+            )}
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
 export function QualityWorkspace() {
   const router = useRouter()
   const searchParams = useSearchParams()
@@ -127,6 +217,8 @@ export function QualityWorkspace() {
   const [selectedSuite, setSelectedSuite] = useState('core')
   const [selectedScenarioId, setSelectedScenarioId] = useState('')
   const [evalBusy, setEvalBusy] = useState<string | null>(null)
+  const [evalEnvironmentPlan, setEvalEnvironmentPlan] = useState<EvalEnvironmentPlan | null>(null)
+  const [evalEnvironmentLoading, setEvalEnvironmentLoading] = useState(false)
   const [approvalBusy, setApprovalBusy] = useState<string | null>(null)
 
   useEffect(() => {
@@ -170,6 +262,27 @@ export function QualityWorkspace() {
     }
   }, [])
 
+  const loadEvalEnvironmentPlan = useCallback(async (opts: { refreshGateway?: boolean } = {}) => {
+    if (!selectedAgentId) {
+      setEvalEnvironmentPlan(null)
+      return
+    }
+    const params = new URLSearchParams({ agentId: selectedAgentId })
+    if (selectedScenarioId) params.set('scenarioId', selectedScenarioId)
+    else if (selectedSuite) params.set('suite', selectedSuite)
+    if (opts.refreshGateway) params.set('refreshGateway', 'true')
+    setEvalEnvironmentLoading(true)
+    try {
+      const plan = await api<EvalEnvironmentPlan>('GET', `/eval/environments?${params.toString()}`, undefined, { timeoutMs: opts.refreshGateway ? 20_000 : 8_000 })
+      setEvalEnvironmentPlan(plan)
+    } catch (err) {
+      setEvalEnvironmentPlan(null)
+      toast.error(err instanceof Error ? err.message : 'Unable to validate eval environment')
+    } finally {
+      setEvalEnvironmentLoading(false)
+    }
+  }, [selectedAgentId, selectedScenarioId, selectedSuite])
+
   useEffect(() => {
     void loadQualityData()
   }, [loadQualityData])
@@ -184,6 +297,10 @@ export function QualityWorkspace() {
     if (!selectedScenarioId && scenarios[0]) setSelectedScenarioId(scenarios[0].id)
   }, [scenarios, selectedScenarioId])
 
+  useEffect(() => {
+    void loadEvalEnvironmentPlan()
+  }, [loadEvalEnvironmentPlan])
+
   useEffect(() => {
     if (!suites.some((suite) => suite.name === selectedSuite) && suites[0]) {
       setSelectedSuite(suites[0].name)
@@ -208,34 +325,56 @@ export function QualityWorkspace() {
       toast.error('Choose an agent and scenario first')
       return
     }
+    if (evalEnvironmentPlan?.status === 'blocked') {
+      toast.error('Fix the validation environment before running this eval')
+      return
+    }
     setEvalBusy(`scenario:${selectedScenarioId}`)
     try {
-      await api<EvalRun>('POST', '/eval/run', { agentId: selectedAgentId, scenarioId: selectedScenarioId }, { timeoutMs: 180_000 })
+      await api<EvalRun>('POST', '/eval/run', {
+        agentId: selectedAgentId,
+        scenarioId: selectedScenarioId,
+        gatewayProfileId: evalEnvironmentPlan?.target?.gatewayProfileId || null,
+        environmentId: evalEnvironmentPlan?.target?.environmentId || null,
+        refreshGateway: evalEnvironmentPlan?.target?.kind === 'gateway',
+      }, { timeoutMs: 180_000 })
       toast.success('Eval scenario completed')
       await loadQualityData({ silent: true })
+      await loadEvalEnvironmentPlan()
     } catch (err) {
       toast.error(err instanceof Error ? err.message : 'Eval scenario failed')
     } finally {
       setEvalBusy(null)
     }
-  }, [loadQualityData, selectedAgentId, selectedScenarioId])
+  }, [evalEnvironmentPlan, loadEvalEnvironmentPlan, loadQualityData, selectedAgentId, selectedScenarioId])
 
   const runSuite = useCallback(async (suiteName: string) => {
     if (!selectedAgentId) {
       toast.error('Choose an agent first')
       return
     }
+    if (evalEnvironmentPlan?.status === 'blocked') {
+      toast.error('Fix the validation environment before running this suite')
+      return
+    }
     setEvalBusy(`suite:${suiteName}`)
     try {
-      const result = await api<EvalSuiteResult>('POST', '/eval/suite', { agentId: selectedAgentId, suite: suiteName }, { timeoutMs: 300_000 })
+      const result = await api<EvalSuiteResult>('POST', '/eval/suite', {
+        agentId: selectedAgentId,
+        suite: suiteName,
+        gatewayProfileId: evalEnvironmentPlan?.target?.gatewayProfileId || null,
+        environmentId: evalEnvironmentPlan?.target?.environmentId || null,
+        refreshGateway: evalEnvironmentPlan?.target?.kind === 'gateway',
+      }, { timeoutMs: 300_000 })
       toast.success(`Suite completed at ${Math.round(result.percentage)}%`)
       await loadQualityData({ silent: true })
+      await loadEvalEnvironmentPlan()
     } catch (err) {
       toast.error(err instanceof Error ? err.message : 'Eval suite failed')
     } finally {
       setEvalBusy(null)
     }
-  }, [loadQualityData, selectedAgentId])
+  }, [evalEnvironmentPlan, loadEvalEnvironmentPlan, loadQualityData, selectedAgentId])
 
   const actOnApproval = useCallback(async (approval: ApprovalRequest, approved: boolean) => {
     setApprovalBusy(approval.id)
@@ -456,6 +595,11 @@ export function QualityWorkspace() {
                       </div>
                     </div>
                   )}
+                  <EvalEnvironmentPanel
+                    plan={evalEnvironmentPlan}
+                    loading={evalEnvironmentLoading}
+                    onRefresh={() => void loadEvalEnvironmentPlan({ refreshGateway: true })}
+                  />
                   <button
                     type="button"
                     onClick={() => openMissionTemplate('release-candidate-qa')}

package/src/lib/server/eval/environment-plan.test.ts

@@ -0,0 +1,221 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+
+import type { Agent, GatewayProfile } from '@/types'
+import { getScenario } from './scenarios'
+import { buildEvalEnvironmentPlan, writeEvalEnvironmentWorkspace } from './environment-plan'
+import type { EvalEnvironmentPlan, EvalScenario } from './types'
+
+function makeAgent(overrides: Partial<Agent> = {}): Agent {
+  return {
+    id: 'agent-1',
+    name: 'Eval Agent',
+    description: 'Validates eval environments.',
+    systemPrompt: 'You are an eval agent.',
+    provider: 'ollama',
+    model: 'llama3',
+    ollamaMode: 'local',
+    tools: [],
+    createdAt: 1,
+    updatedAt: 1,
+    ...overrides,
+  } as Agent
+}
+
+function makeGateway(overrides: Partial<GatewayProfile> = {}): GatewayProfile {
+  return {
+    id: 'gateway-1',
+    name: 'Gateway 1',
+    provider: 'openclaw',
+    endpoint: 'http://127.0.0.1:18789/v1',
+    wsUrl: 'ws://127.0.0.1:18789',
+    credentialId: null,
+    status: 'healthy',
+    stats: {
+      nodeCount: 1,
+      connectedNodeCount: 1,
+      environmentCount: 1,
+      availableEnvironmentCount: 1,
+      pendingNodePairings: 0,
+      pendingDevicePairings: 0,
+      pairedDeviceCount: 0,
+      lastTopologyCheckedAt: 2,
+      lastTopologyErrorCount: 0,
+      lastTopologyError: null,
+    },
+    createdAt: 1,
+    updatedAt: 1,
+    ...overrides,
+  } as GatewayProfile
+}
+
+test('eval environment plan blocks missing CLI provider readiness before a run spends tokens', async () => {
+  const plan = await buildEvalEnvironmentPlan(
+    { agentId: 'agent-cli', scenarioId: 'coding-prime' },
+    {
+      now: () => 123,
+      loadAgents: () => ({
+        'agent-cli': makeAgent({
+          id: 'agent-cli',
+          provider: 'codex-cli',
+          model: 'gpt-5.2',
+          ollamaMode: null,
+        }),
+      }),
+      listGatewayProfiles: () => [],
+      checkCliProviderReady: () => ({
+        ok: false,
+        message: 'Codex CLI is not installed.',
+        providerId: 'codex-cli',
+        displayName: 'Codex CLI',
+        binaryName: 'codex',
+      }),
+    },
+  )
+
+  assert.equal(plan.status, 'blocked')
+  assert.equal(plan.target?.kind, 'local')
+  assert.ok(plan.checks.some((check) => check.code === 'cli_provider_not_ready' && check.level === 'error'))
+})
+
+test('eval environment plan refreshes gateway environments and selects an available target', async () => {
+  const plan = await buildEvalEnvironmentPlan(
+    {
+      agentId: 'agent-openclaw',
+      scenarioId: 'coding-prime',
+      refreshGateway: true,
+    },
+    {
+      now: () => 456,
+      loadAgents: () => ({
+        'agent-openclaw': makeAgent({
+          id: 'agent-openclaw',
+          provider: 'openclaw',
+          model: 'default',
+          gatewayProfileId: 'gateway-1',
+        }),
+      }),
+      listGatewayProfiles: () => [makeGateway()],
+      listGatewayEnvironments: async () => ({
+        profile: makeGateway(),
+        connected: true,
+        refreshedAt: 789,
+        errors: [],
+        environments: [
+          { id: 'env-busy', type: 'sandbox', label: 'Busy', status: 'starting', capabilities: ['agent.run'] },
+          { id: 'env-ready', type: 'sandbox', label: 'Ready', status: 'available', capabilities: ['agent.run', 'workspace'] },
+        ],
+      }),
+    },
+  )
+
+  assert.equal(plan.status, 'ready')
+  assert.equal(plan.target?.kind, 'gateway')
+  assert.equal(plan.target?.environmentId, 'env-ready')
+  assert.equal(plan.target?.environmentStatus, 'available')
+  assert.deepEqual(plan.target?.capabilities, ['agent.run', 'workspace'])
+  assert.ok(plan.checks.some((check) => check.code === 'environment_available'))
+})
+
+test('eval environment plan blocks gateways with no available execution environments', async () => {
+  const plan = await buildEvalEnvironmentPlan(
+    { agentId: 'agent-openclaw', scenarioId: 'coding-prime' },
+    {
+      loadAgents: () => ({
+        'agent-openclaw': makeAgent({
+          id: 'agent-openclaw',
+          provider: 'openclaw',
+          model: 'default',
+          gatewayProfileId: 'gateway-1',
+        }),
+      }),
+      listGatewayProfiles: () => [
+        makeGateway({
+          stats: {
+            nodeCount: 1,
+            connectedNodeCount: 1,
+            environmentCount: 2,
+            availableEnvironmentCount: 0,
+            pendingNodePairings: 0,
+            pendingDevicePairings: 0,
+            pairedDeviceCount: 0,
+          },
+        }),
+      ],
+    },
+  )
+
+  assert.equal(plan.status, 'blocked')
+  assert.ok(plan.checks.some((check) => check.code === 'no_available_gateway_environment'))
+})
+
+test('eval workspace writer materializes manifests, env hints, and scenario fixtures', async () => {
+  const scenario = getScenario('multi-step-analyze')
+  assert.ok(scenario)
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-eval-env-'))
+  const plan = await buildEvalEnvironmentPlan(
+    { agentId: 'agent-1', scenarioId: scenario.id },
+    {
+      now: () => 999,
+      loadAgents: () => ({ 'agent-1': makeAgent() }),
+      listGatewayProfiles: () => [],
+    },
+  )
+
+  const files = writeEvalEnvironmentWorkspace({
+    runId: 'run-1',
+    workspacePath: root,
+    scenario,
+    plan,
+  })
+
+  assert.ok(files.some((file) => file.path === 'package.json' && file.kind === 'fixture'))
+  assert.ok(fs.existsSync(path.join(root, 'README.md')))
+  assert.ok(fs.existsSync(path.join(root, 'environment.json')))
+  assert.ok(fs.existsSync(path.join(root, '.env.swarmclaw-eval')))
+  const fixture = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')) as { dependencies?: Record<string, string> }
+  assert.equal(fixture.dependencies?.zod, '^4.1.13')
+  assert.ok(fs.readFileSync(path.join(root, '.env.swarmclaw-eval'), 'utf8').includes('SWARMCLAW_EVAL_RUN_ID="run-1"'))
+})
+
+test('eval workspace writer refuses fixture paths outside the eval workspace', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-eval-unsafe-'))
+  const scenario: EvalScenario = {
+    id: 'unsafe-fixture',
+    name: 'Unsafe Fixture',
+    category: 'coding',
+    description: 'Unsafe fixture path test',
+    userMessage: 'noop',
+    expectedBehaviors: [],
+    scoringCriteria: [],
+    timeoutMs: 1,
+    tools: [],
+    fixtures: [{ path: '../outside.txt', content: 'nope' }],
+  }
+  const plan: EvalEnvironmentPlan = {
+    generatedAt: 1,
+    status: 'ready',
+    agentId: 'agent-1',
+    agentName: 'Eval Agent',
+    scenarioIds: [scenario.id],
+    suite: null,
+    target: null,
+    checks: [],
+    requiredTools: [],
+    missingTools: [],
+    maxScore: 0,
+    timeoutMs: 1,
+    generatedFiles: [],
+    envHints: [],
+  }
+
+  assert.throws(() => writeEvalEnvironmentWorkspace({
+    runId: 'run-unsafe',
+    workspacePath: root,
+    scenario,
+    plan,
+  }), /Unsafe eval fixture path/)
+})

package/src/lib/server/eval/environment-plan.ts

@@ -0,0 +1,498 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { WORKSPACE_DIR } from '@/lib/server/data-dir'
+import { resolveAgentRouteCandidatesWithProfiles, type ResolvedAgentRoute } from '@/lib/server/agents/agent-runtime-config'
+import { checkCliProviderReady, type CliProviderReadyResult } from '@/lib/server/cli-provider-readiness'
+import { listOpenClawGatewayEnvironments } from '@/lib/server/gateways/gateway-topology'
+import { loadAgents, loadCredentials } from '@/lib/server/storage'
+import { isCliProviderId } from '@/lib/providers/cli-provider-metadata'
+import type { Agent, GatewayProfile, OpenClawEnvironmentSummary, OpenClawGatewayEnvironmentList } from '@/types'
+import type {
+  EvalEnvironmentCheck,
+  EvalEnvironmentGeneratedFile,
+  EvalEnvironmentPlan,
+  EvalEnvironmentTarget,
+  EvalScenario,
+  EvalScenarioFixture,
+} from './types'
+import { getScenario, getSuiteScenarios } from './scenarios'
+import { listOpenClawGatewayProfiles } from '../gateways/gateway-profile-service'
+
+export interface EvalEnvironmentPlanInput {
+  agentId: string
+  scenarioId?: string | null
+  suite?: string | null
+  gatewayProfileId?: string | null
+  environmentId?: string | null
+  refreshGateway?: boolean
+}
+
+interface EvalEnvironmentPlanDeps {
+  now?: () => number
+  loadAgents?: () => Record<string, Agent>
+  loadCredentials?: () => Record<string, unknown>
+  listGatewayProfiles?: () => GatewayProfile[]
+  listGatewayEnvironments?: (id: string) => Promise<OpenClawGatewayEnvironmentList | null>
+  checkCliProviderReady?: (providerId: string) => CliProviderReadyResult
+}
+
+interface WriteEvalWorkspaceOptions {
+  runId: string
+  workspacePath: string
+  scenario: EvalScenario
+  plan: EvalEnvironmentPlan
+}
+
+function normalizeOptionalId(value: string | null | undefined): string | null {
+  return typeof value === 'string' && value.trim() ? value.trim() : null
+}
+
+function uniqueStrings(values: string[]): string[] {
+  return [...new Set(values.map((value) => value.trim()).filter(Boolean))]
+    .sort((left, right) => left.localeCompare(right))
+}
+
+function scenarioSet(input: EvalEnvironmentPlanInput): { scenarios: EvalScenario[]; missing?: string } {
+  const scenarioId = normalizeOptionalId(input.scenarioId)
+  if (scenarioId) {
+    const scenario = getScenario(scenarioId)
+    return scenario ? { scenarios: [scenario] } : { scenarios: [], missing: scenarioId }
+  }
+  const suite = normalizeOptionalId(input.suite) || 'core'
+  return { scenarios: getSuiteScenarios(suite) }
+}
+
+function maxScore(scenarios: EvalScenario[]): number {
+  return scenarios.reduce(
+    (sum, scenario) => sum + scenario.scoringCriteria.reduce((criterionSum, criterion) => criterionSum + criterion.weight, 0),
+    0,
+  )
+}
+
+function timeoutMs(scenarios: EvalScenario[]): number {
+  return scenarios.reduce((sum, scenario) => sum + scenario.timeoutMs, 0)
+}
+
+function fixtureFiles(scenarios: EvalScenario[]): EvalEnvironmentGeneratedFile[] {
+  return scenarios.flatMap((scenario) => (scenario.fixtures || []).map((fixture) => ({
+    path: fixture.path,
+    kind: 'fixture' as const,
+    required: true,
+  })))
+}
+
+function baseGeneratedFiles(scenarios: EvalScenario[]): EvalEnvironmentGeneratedFile[] {
+  return [
+    { path: 'README.md', kind: 'readme', required: true },
+    { path: 'environment.json', kind: 'manifest', required: true },
+    { path: '.env.swarmclaw-eval', kind: 'env', required: true },
+    ...fixtureFiles(scenarios),
+  ]
+}
+
+function providerNeedsCredential(route: ResolvedAgentRoute): boolean {
+  if (route.provider === 'openclaw') return false
+  if (route.provider === 'ollama' && route.ollamaMode !== 'cloud') return false
+  if (isCliProviderId(route.provider)) return false
+  return true
+}
+
+function credentialExists(credentialId: string | null | undefined, credentials: Record<string, unknown>): boolean {
+  return typeof credentialId === 'string' && credentialId.trim() ? Boolean(credentials[credentialId]) : false
+}
+
+function checkLevelRank(level: EvalEnvironmentCheck['level']): number {
+  if (level === 'error') return 2
+  if (level === 'warn') return 1
+  return 0
+}
+
+function statusFromChecks(checks: EvalEnvironmentCheck[]): EvalEnvironmentPlan['status'] {
+  const max = checks.reduce((rank, check) => Math.max(rank, checkLevelRank(check.level)), 0)
+  if (max >= 2) return 'blocked'
+  if (max >= 1) return 'warning'
+  return 'ready'
+}
+
+function pickGatewayProfile(
+  route: ResolvedAgentRoute | null,
+  profiles: GatewayProfile[],
+  requestedProfileId: string | null,
+): GatewayProfile | null {
+  if (requestedProfileId) {
+    return profiles.find((profile) => profile.id === requestedProfileId) || null
+  }
+  if (route?.gatewayProfileId) {
+    return profiles.find((profile) => profile.id === route.gatewayProfileId) || null
+  }
+  return profiles.find((profile) => profile.isDefault) || profiles[0] || null
+}
+
+function summarizeGatewayTarget(route: ResolvedAgentRoute, profile: GatewayProfile | null): EvalEnvironmentTarget {
+  return {
+    kind: 'gateway',
+    provider: route.provider,
+    model: route.model,
+    label: profile?.name || route.label,
+    gatewayProfileId: profile?.id || route.gatewayProfileId || null,
+    capabilities: ['agent.run', 'sessions', 'tools', 'workspace'],
+    refreshedAt: profile?.stats?.lastTopologyCheckedAt || profile?.lastCheckedAt || null,
+  }
+}
+
+function summarizeLocalTarget(route: ResolvedAgentRoute): EvalEnvironmentTarget {
+  return {
+    kind: 'local',
+    provider: route.provider,
+    model: route.model,
+    label: route.label,
+    capabilities: ['agent.run', 'tools', 'workspace'],
+    refreshedAt: null,
+  }
+}
+
+function addEnvHint(
+  hints: EvalEnvironmentPlan['envHints'],
+  key: string,
+  value: string | null | undefined,
+  description?: string,
+): void {
+  if (!value) return
+  hints.push({ key, value, ...(description ? { description } : {}) })
+}
+
+function buildEnvHints(params: {
+  agent: Agent | null
+  scenarios: EvalScenario[]
+  suite: string | null
+  target: EvalEnvironmentTarget | null
+}): EvalEnvironmentPlan['envHints'] {
+  const hints: EvalEnvironmentPlan['envHints'] = []
+  addEnvHint(hints, 'SWARMCLAW_EVAL_AGENT_ID', params.agent?.id, 'Agent under validation')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_AGENT_NAME', params.agent?.name, 'Agent display name')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_SCENARIOS', params.scenarios.map((scenario) => scenario.id).join(','), 'Comma-separated eval scenario ids')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_SUITE', params.suite, 'Eval suite name')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_TARGET_KIND', params.target?.kind, 'Resolved execution target kind')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_PROVIDER', params.target?.provider, 'Resolved provider')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_MODEL', params.target?.model, 'Resolved model')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_GATEWAY_PROFILE_ID', params.target?.gatewayProfileId || null, 'Resolved gateway profile id')
+  addEnvHint(hints, 'SWARMCLAW_EVAL_ENVIRONMENT_ID', params.target?.environmentId || null, 'Requested or selected gateway environment id')
+  return hints
+}
+
+function normalizeEnvironmentCapabilities(environment: OpenClawEnvironmentSummary | null | undefined): string[] {
+  return uniqueStrings(environment?.capabilities || [])
+}
+
+async function attachGatewayEnvironment(
+  target: EvalEnvironmentTarget,
+  profile: GatewayProfile | null,
+  checks: EvalEnvironmentCheck[],
+  input: EvalEnvironmentPlanInput,
+  deps: Required<Pick<EvalEnvironmentPlanDeps, 'listGatewayEnvironments'>>,
+): Promise<EvalEnvironmentTarget> {
+  if (!profile) return target
+  const requestedEnvironmentId = normalizeOptionalId(input.environmentId)
+
+  if (profile.status === 'offline') {
+    checks.push({
+      code: 'gateway_offline',
+      level: 'error',
+      message: `${profile.name} is offline.`,
+      hint: 'Refresh or repair the gateway before running evals through it.',
+    })
+  } else if (profile.status === 'degraded') {
+    checks.push({
+      code: 'gateway_degraded',
+      level: 'warn',
+      message: `${profile.name} is degraded.`,
+      detail: profile.lastError || undefined,
+    })
+  } else if (profile.status === 'pending' || profile.status === 'unknown') {
+    checks.push({
+      code: 'gateway_unverified',
+      level: 'warn',
+      message: `${profile.name} has not reported a healthy gateway status yet.`,
+    })
+  }
+
+  const environmentCount = profile.stats?.environmentCount || 0
+  const availableEnvironmentCount = profile.stats?.availableEnvironmentCount || 0
+  if (environmentCount > 0 && availableEnvironmentCount === 0) {
+    checks.push({
+      code: 'no_available_gateway_environment',
+      level: 'error',
+      message: `${profile.name} has ${environmentCount} execution environment${environmentCount === 1 ? '' : 's'}, but none are available.`,
+    })
+  }
+
+  if (!input.refreshGateway) {
+    if (requestedEnvironmentId) {
+      checks.push({
+        code: 'environment_not_refreshed',
+        level: 'warn',
+        message: `Environment ${requestedEnvironmentId} was requested but not refreshed.`,
+        hint: 'Run validation with refresh enabled to verify the exact environment.',
+      })
+      return { ...target, environmentId: requestedEnvironmentId }
+    }
+    checks.push({
+      code: 'gateway_snapshot_only',
+      level: 'info',
+      message: 'Using the last stored gateway topology snapshot for validation.',
+    })
+    return target
+  }
+
+  const snapshot = await deps.listGatewayEnvironments(profile.id)
+  if (!snapshot) {
+    checks.push({
+      code: 'gateway_environment_snapshot_missing',
+      level: 'error',
+      message: `${profile.name} could not be refreshed for environment validation.`,
+    })
+    return target
+  }
+  for (const error of snapshot.errors) {
+    checks.push({
+      code: 'gateway_environment_refresh_error',
+      level: 'warn',
+      message: `${error.method}: ${error.message}`,
+    })
+  }
+  const environments = snapshot.environments
+  const selected = requestedEnvironmentId
+    ? environments.find((environment) => environment.id === requestedEnvironmentId) || null
+    : environments.find((environment) => environment.status === 'available') || environments[0] || null
+  if (requestedEnvironmentId && !selected) {
+    checks.push({
+      code: 'environment_not_found',
+      level: 'error',
+      message: `Requested execution environment ${requestedEnvironmentId} was not found on ${profile.name}.`,
+    })
+    return { ...target, environmentId: requestedEnvironmentId, refreshedAt: snapshot.refreshedAt }
+  }
+  if (!selected) {
+    checks.push({
+      code: 'no_gateway_environments',
+      level: 'warn',
+      message: `${profile.name} did not report any execution environments.`,
+    })
+    return { ...target, refreshedAt: snapshot.refreshedAt }
+  }
+  if (selected.status !== 'available') {
+    checks.push({
+      code: 'environment_unavailable',
+      level: selected.status === 'error' ? 'error' : 'warn',
+      message: `${selected.label || selected.id} is ${selected.status}.`,
+    })
+  } else {
+    checks.push({
+      code: 'environment_available',
+      level: 'info',
+      message: `${selected.label || selected.id} is available for validation runs.`,
+    })
+  }
+  return {
+    ...target,
+    environmentId: selected.id,
+    environmentLabel: selected.label || selected.id,
+    environmentStatus: selected.status,
+    capabilities: normalizeEnvironmentCapabilities(selected),
+    refreshedAt: snapshot.refreshedAt,
+  }
+}
+
+export async function buildEvalEnvironmentPlan(
+  input: EvalEnvironmentPlanInput,
+  deps: EvalEnvironmentPlanDeps = {},
+): Promise<EvalEnvironmentPlan> {
+  const now = deps.now || (() => Date.now())
+  const generatedAt = now()
+  const loadAgentsImpl = deps.loadAgents || (() => loadAgents() as Record<string, Agent>)
+  const loadCredentialsImpl = deps.loadCredentials || (() => loadCredentials() as Record<string, unknown>)
+  const listGatewayProfilesImpl = deps.listGatewayProfiles || listOpenClawGatewayProfiles
+  const checkCliProviderReadyImpl = deps.checkCliProviderReady || checkCliProviderReady
+  const checks: EvalEnvironmentCheck[] = []
+  const { scenarios, missing } = scenarioSet(input)
+  const suite = normalizeOptionalId(input.suite) || (input.scenarioId ? null : 'core')
+  const agents = loadAgentsImpl()
+  const agent = agents[input.agentId] || null
+  const requiredTools = uniqueStrings(scenarios.flatMap((scenario) => scenario.tools || []))
+  let target: EvalEnvironmentTarget | null = null
+
+  if (missing) {
+    checks.push({
+      code: 'scenario_not_found',
+      level: 'error',
+      message: `Eval scenario ${missing} was not found.`,
+    })
+  } else if (scenarios.length === 0) {
+    checks.push({
+      code: 'scenario_set_empty',
+      level: 'error',
+      message: 'No eval scenarios matched the requested suite.',
+    })
+  }
+
+  if (!agent) {
+    checks.push({
+      code: 'agent_not_found',
+      level: 'error',
+      message: `Agent ${input.agentId} was not found.`,
+    })
+  } else {
+    if (agent.trashedAt) {
+      checks.push({ code: 'agent_trashed', level: 'error', message: `${agent.name} is in trash.` })
+    }
+    if (agent.disabled) {
+      checks.push({ code: 'agent_disabled', level: 'error', message: `${agent.name} is disabled.` })
+    }
+
+    const gatewayProfiles = listGatewayProfilesImpl()
+    const [route] = resolveAgentRouteCandidatesWithProfiles(agent, gatewayProfiles)
+    if (!route) {
+      checks.push({
+        code: 'route_unresolved',
+        level: 'error',
+        message: `${agent.name} does not have a runnable provider/model route.`,
+      })
+    } else if (route.provider === 'openclaw') {
+      const profile = pickGatewayProfile(route, gatewayProfiles, normalizeOptionalId(input.gatewayProfileId))
+      if (!profile) {
+        checks.push({
+          code: 'gateway_profile_missing',
+          level: 'error',
+          message: 'No gateway profile is available for this agent route.',
+        })
+        target = summarizeGatewayTarget(route, null)
+      } else {
+        target = await attachGatewayEnvironment(
+          summarizeGatewayTarget(route, profile),
+          profile,
+          checks,
+          input,
+          { listGatewayEnvironments: deps.listGatewayEnvironments || listOpenClawGatewayEnvironments },
+        )
+      }
+    } else {
+      target = summarizeLocalTarget(route)
+      if (isCliProviderId(route.provider)) {
+        const ready = checkCliProviderReadyImpl(route.provider)
+        checks.push({
+          code: ready.ok ? 'cli_provider_ready' : 'cli_provider_not_ready',
+          level: ready.ok ? 'info' : 'error',
+          message: ready.message,
+          detail: ready.binaryPath,
+        })
+      } else if (providerNeedsCredential(route) && !credentialExists(route.credentialId, loadCredentialsImpl())) {
+        checks.push({
+          code: 'credential_missing',
+          level: 'warn',
+          message: `${route.provider} does not have a stored credential for this route.`,
+          hint: 'The run may still work if the provider is configured through environment variables.',
+        })
+      }
+    }
+  }
+
+  if (requiredTools.length > 0) {
+    checks.push({
+      code: 'tools_declared',
+      level: 'info',
+      message: `${requiredTools.length} eval tool${requiredTools.length === 1 ? '' : 's'} will be enabled: ${requiredTools.join(', ')}.`,
+    })
+  } else {
+    checks.push({
+      code: 'no_tools_required',
+      level: 'info',
+      message: 'This eval scenario does not require tool access.',
+    })
+  }
+
+  const envHints = buildEnvHints({ agent, scenarios, suite, target })
+
+  return {
+    generatedAt,
+    status: statusFromChecks(checks),
+    agentId: input.agentId,
+    agentName: agent?.name || input.agentId,
+    scenarioIds: scenarios.map((scenario) => scenario.id),
+    suite,
+    target,
+    checks,
+    requiredTools,
+    missingTools: [],
+    maxScore: maxScore(scenarios),
+    timeoutMs: timeoutMs(scenarios),
+    generatedFiles: baseGeneratedFiles(scenarios),
+    envHints,
+  }
+}
+
+function safeFixtureDestination(workspacePath: string, fixture: EvalScenarioFixture): string {
+  const relative = fixture.path.trim()
+  if (!relative || path.isAbsolute(relative)) {
+    throw new Error(`Unsafe eval fixture path: ${fixture.path}`)
+  }
+  const destination = path.resolve(workspacePath, relative)
+  const root = path.resolve(workspacePath)
+  if (destination !== root && !destination.startsWith(`${root}${path.sep}`)) {
+    throw new Error(`Unsafe eval fixture path: ${fixture.path}`)
+  }
+  return destination
+}
+
+function writeTextFile(filePath: string, content: string, mode?: number): void {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true })
+  fs.writeFileSync(filePath, content.endsWith('\n') ? content : `${content}\n`, { encoding: 'utf8', mode })
+}
+
+function envLine(hint: EvalEnvironmentPlan['envHints'][number]): string {
+  return `${hint.key}=${JSON.stringify(hint.value)}`
+}
+
+export function writeEvalEnvironmentWorkspace(options: WriteEvalWorkspaceOptions): EvalEnvironmentGeneratedFile[] {
+  const { runId, workspacePath, scenario, plan } = options
+  fs.mkdirSync(workspacePath, { recursive: true })
+
+  const readme = [
+    `# Eval Workspace: ${scenario.name}`,
+    '',
+    `Run ID: ${runId}`,
+    `Agent: ${plan.agentName} (${plan.agentId})`,
+    `Scenario: ${scenario.id}`,
+    `Status at start: ${plan.status}`,
+    '',
+    'Runtime manifest: ./environment.json',
+    'Environment hints: ./.env.swarmclaw-eval',
+    '',
+    'This directory is isolated for eval artifacts, fixtures, and generated outputs.',
+  ].join('\n')
+  writeTextFile(path.join(workspacePath, 'README.md'), readme)
+  writeTextFile(path.join(workspacePath, 'environment.json'), JSON.stringify({ runId, plan }, null, 2))
+  writeTextFile(
+    path.join(workspacePath, '.env.swarmclaw-eval'),
+    [
+      '# Generated by SwarmClaw. Contains eval context only, not secrets.',
+      `SWARMCLAW_EVAL_RUN_ID=${JSON.stringify(runId)}`,
+      ...plan.envHints.map(envLine),
+    ].join('\n'),
+  )
+
+  for (const fixture of scenario.fixtures || []) {
+    writeTextFile(safeFixtureDestination(workspacePath, fixture), fixture.content, fixture.mode)
+  }
+
+  return [
+    { path: 'README.md', kind: 'readme', required: true },
+    { path: 'environment.json', kind: 'manifest', required: true },
+    { path: '.env.swarmclaw-eval', kind: 'env', required: true },
+    ...fixtureFiles([scenario]),
+  ]
+}
+
+export function resolveEvalWorkspacePath(runId: string): string {
+  return path.join(WORKSPACE_DIR, 'evals', runId)
+}

package/src/lib/server/eval/runner.ts

@@ -10,6 +10,7 @@ import { executeExecutionChatTurn } from '@/lib/server/execution-engine/chat-tur
 import { WORKSPACE_DIR } from '../data-dir'
 import type { Session } from '@/types'
 import { errorMessage } from '@/lib/shared-utils'
+import { buildEvalEnvironmentPlan, writeEvalEnvironmentWorkspace } from './environment-plan'
 
 export function resolveEvalSessionCwd(runId: string): string {
   const dir = path.join(WORKSPACE_DIR, 'evals', runId)
@@ -17,7 +18,17 @@ export function resolveEvalSessionCwd(runId: string): string {
   return dir
 }
 
-export async function runEvalScenario(scenarioId: string, agentId: string): Promise<EvalRun> {
+export interface RunEvalScenarioOptions {
+  gatewayProfileId?: string | null
+  environmentId?: string | null
+  refreshGateway?: boolean
+}
+
+export async function runEvalScenario(
+  scenarioId: string,
+  agentId: string,
+  options: RunEvalScenarioOptions = {},
+): Promise<EvalRun> {
   const scenario = getScenario(scenarioId)
   if (!scenario) throw new Error(`Unknown eval scenario: ${scenarioId}`)
 
@@ -29,6 +40,13 @@ export async function runEvalScenario(scenarioId: string, agentId: string): Prom
   const sessionId = `eval-${runId}`
   const now = Date.now()
   const sessionCwd = resolveEvalSessionCwd(runId)
+  const environment = await buildEvalEnvironmentPlan({
+    agentId,
+    scenarioId,
+    gatewayProfileId: options.gatewayProfileId || null,
+    environmentId: options.environmentId || null,
+    refreshGateway: options.refreshGateway === true,
+  })
 
   const run: EvalRun = {
     id: runId,
@@ -40,6 +58,34 @@ export async function runEvalScenario(scenarioId: string, agentId: string): Prom
     maxScore: scenario.scoringCriteria.reduce((sum, c) => sum + c.weight, 0),
     details: [],
     sessionId,
+    environment,
+  }
+
+  writeEvalEnvironmentWorkspace({
+    runId,
+    workspacePath: sessionCwd,
+    scenario,
+    plan: environment,
+  })
+
+  if (environment.status === 'blocked') {
+    run.status = 'failed'
+    run.error = environment.checks
+      .filter((check) => check.level === 'error')
+      .map((check) => check.message)
+      .join(' ')
+      || 'Eval environment validation failed.'
+    run.endedAt = Date.now()
+    run.details = environment.checks
+      .filter((check) => check.level !== 'info')
+      .map((check) => ({
+        criterion: check.code,
+        score: 0,
+        maxScore: 0,
+        evidence: check.message,
+      }))
+    saveEvalRun(run)
+    return run
   }
 
   // Create temporary eval session
@@ -114,7 +160,7 @@ export async function runEvalScenario(scenarioId: string, agentId: string): Prom
 
 export async function runEvalSuite(
   agentId: string,
-  opts: { categories?: string[]; suite?: string } = {},
+  opts: { categories?: string[]; suite?: string; gatewayProfileId?: string | null; environmentId?: string | null; refreshGateway?: boolean } = {},
 ): Promise<EvalSuiteResult> {
   let scenarios: EvalScenario[]
   if (opts.suite) {
@@ -130,7 +176,11 @@ export async function runEvalSuite(
 
   const runs: EvalRun[] = []
   for (const scenario of scenarios) {
-    const evalRun = await runEvalScenario(scenario.id, agentId)
+    const evalRun = await runEvalScenario(scenario.id, agentId, {
+      gatewayProfileId: opts.gatewayProfileId || null,
+      environmentId: opts.environmentId || null,
+      refreshGateway: opts.refreshGateway === true,
+    })
     runs.push(evalRun)
   }
 

package/src/lib/server/eval/scenarios.ts

@@ -212,6 +212,24 @@ const CORE_SCENARIOS: EvalScenario[] = [
     ],
     timeoutMs: 60_000,
     tools: ['shell', 'files'],
+    fixtures: [
+      {
+        path: 'package.json',
+        content: JSON.stringify({
+          name: 'swarmclaw-eval-fixture',
+          version: '0.0.0',
+          private: true,
+          dependencies: {
+            '@modelcontextprotocol/sdk': '^1.29.0',
+            zod: '^4.1.13',
+          },
+          devDependencies: {
+            typescript: '^5.9.3',
+            tsx: '^4.20.6',
+          },
+        }, null, 2),
+      },
+    ],
   },
 ]
 

package/src/lib/server/eval/types.ts

@@ -17,10 +17,64 @@ export interface EvalScenario {
   scoringCriteria: ScoringCriterion[]
   timeoutMs: number
   tools: string[]
+  fixtures?: EvalScenarioFixture[]
   /** Optional suite tag. Scenarios without a suite belong to the 'core' suite. */
   suite?: EvalSuite
 }
 
+export interface EvalScenarioFixture {
+  path: string
+  content: string
+  mode?: number
+}
+
+export type EvalEnvironmentStatus = 'ready' | 'warning' | 'blocked'
+export type EvalEnvironmentCheckLevel = 'info' | 'warn' | 'error'
+
+export interface EvalEnvironmentCheck {
+  code: string
+  level: EvalEnvironmentCheckLevel
+  message: string
+  detail?: string
+  hint?: string
+}
+
+export interface EvalEnvironmentTarget {
+  kind: 'local' | 'gateway'
+  provider: string
+  model: string
+  label: string
+  gatewayProfileId?: string | null
+  environmentId?: string | null
+  environmentLabel?: string | null
+  environmentStatus?: string | null
+  capabilities?: string[]
+  refreshedAt?: number | null
+}
+
+export interface EvalEnvironmentGeneratedFile {
+  path: string
+  kind: 'readme' | 'manifest' | 'env' | 'fixture'
+  required: boolean
+}
+
+export interface EvalEnvironmentPlan {
+  generatedAt: number
+  status: EvalEnvironmentStatus
+  agentId: string
+  agentName: string
+  scenarioIds: string[]
+  suite?: string | null
+  target: EvalEnvironmentTarget | null
+  checks: EvalEnvironmentCheck[]
+  requiredTools: string[]
+  missingTools: string[]
+  maxScore: number
+  timeoutMs: number
+  generatedFiles: EvalEnvironmentGeneratedFile[]
+  envHints: Array<{ key: string; value: string; description?: string }>
+}
+
 export interface EvalRun {
   id: string
   scenarioId: string
@@ -32,6 +86,7 @@ export interface EvalRun {
   maxScore: number
   details: EvalCriterionResult[]
   sessionId?: string
+  environment?: EvalEnvironmentPlan
   error?: string
 }