actionagent 1.3.0 → 1.5.0

data/app/services/action_agent/playwright_mcp_client.rb

@@ -1,20 +1,10 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  require "net/http"
-  require "resolv"
-
-  # Minimal MCP client (streamable HTTP transport) for a Playwright MCP
-  # server — typically `npx @playwright/mcp --port 8931` running beside the
-  # app in development, or a sandbox-provisioned browser container in
-  # production. Speaks just enough JSON-RPC for tools/call: initialize once
-  # per process, then call tools under the session id the server hands back.
-  class PlaywrightMCPClient
+  # The Playwright MCP server as one configured MCPClient: a process-wide
+  # instance pointed at the url PLAYWRIGHT_MCP_URL names.
+  class PlaywrightMCPClient < MCPClient
     DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
-    OPEN_TIMEOUT_SECONDS = 5
-    READ_TIMEOUT_SECONDS = 60
-
-    class Error < StandardError; end
 
     def self.instance
       @instance ||= new
@@ -25,124 +15,19 @@ module ActionAgent
     end
 
     def initialize(url: DEFAULT_URL)
-      @uri = URI(url)
-      @mutex = Mutex.new
+      super(url: url, label: "Playwright")
     end
 
-    # Returns { text:, is_error: } — the tool result's text content.
+    # Restarting the shared instance on an unreachable server is this
+    # subclass's concern: the next call re-initializes rather than reusing a
+    # session the server has forgotten. The message names the local fix.
     def call_tool(name, arguments = {})
-      Rails.logger.debug("[PlaywrightMCPClient] call #{name} args=#{arguments.inspect[0, 200]}")
-      ensure_session!
-      response = post(
-        { jsonrpc: "2.0", id: next_id, method: "tools/call",
-          params: { name: name, arguments: arguments } },
-        session: @session_id
-      )
-      result = response["result"]
-      unless result
-        Rails.logger.warn("[PlaywrightMCPClient] #{name} unexpected response: #{response.inspect[0, 500]}")
-        raise Error, (response.dig("error", "message") || "empty MCP response")
-      end
-
-      text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
-      { text: text, is_error: result["isError"] ? true : false }
-    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
+      super
+    rescue MCPClient::Error => e
       self.class.reset!
-      raise Error, "Playwright MCP server unreachable at #{@uri} (#{e.class}): start it with `npx @playwright/mcp --port 8931`"
-    end
-
-    private
-
-    def ensure_session!
-      @mutex.synchronize do
-        next if @session_id
-
-        _body, response = post_raw(
-          { jsonrpc: "2.0", id: next_id, method: "initialize",
-            params: { protocolVersion: "2025-03-26", capabilities: {},
-                      clientInfo: { name: "activeagents", version: "1.0" } } }
-        )
-        @session_id = response["mcp-session-id"]
-        raise Error, "MCP server did not return a session id" unless @session_id
-
-        post({ jsonrpc: "2.0", method: "notifications/initialized" }, session: @session_id)
-      end
-    end
-
-    def post(payload, session: nil)
-      body, _response = post_raw(payload, session: session)
-      body
-    end
-
-    def post_raw(payload, session: nil)
-      # Tool calls run inside the provider SDK's streaming enumerator — a
-      # fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
-      # body comes back empty). A dedicated thread always does real blocking
-      # IO outside any fiber/scheduler context.
-      Thread.new { blocking_post_raw(payload, session: session) }.value
-    end
-
-    def blocking_post_raw(payload, session: nil)
-      http = Net::HTTP.new(@uri.host, @uri.port)
-      # Container->host bridge hostnames (host.orb.internal) publish an IPv6
-      # address whose path doesn't reach the server; dual-stack connects then
-      # fail intermittently. Pin to IPv4 while keeping the Host header.
-      if (ipv4 = ipv4_address)
-        http.ipaddr = ipv4
-      end
-      http.open_timeout = OPEN_TIMEOUT_SECONDS
-      http.read_timeout = READ_TIMEOUT_SECONDS
-      request = Net::HTTP::Post.new(@uri.request_uri)
-      request["Content-Type"] = "application/json"
-      request["Accept"] = "application/json, text/event-stream"
-      request["Mcp-Session-Id"] = session if session
-      request.body = payload.to_json
-
-      response = http.request(request)
-      Rails.logger.debug(
-        "[PlaywrightMCPClient] #{payload[:method]} -> #{response.code} " \
-        "ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
-      )
-      unless response.code.to_i.between?(200, 299)
-        Rails.logger.warn("[PlaywrightMCPClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
-        raise Error, "MCP server returned HTTP #{response.code}"
-      end
-
-      parsed = parse_body(response)
-      if parsed.empty? && payload[:id]
-        Rails.logger.warn("[PlaywrightMCPClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
-      end
-      [ parsed, response ]
-    end
-
-    # Streamable HTTP answers as plain JSON or as an SSE stream whose data:
-    # lines carry the JSON-RPC response — accept both.
-    def parse_body(response)
-      body = response.body.to_s
-      return {} if body.empty?
-
-      if response["Content-Type"].to_s.include?("text/event-stream")
-        body.lines
-            .select { |line| line.start_with?("data:") }
-            .filter_map { |line| JSON.parse(line.delete_prefix("data:").strip) rescue nil }
-            .find { |json| json["result"] || json["error"] } || {}
-      else
-        JSON.parse(body)
-      end
-    rescue JSON::ParserError
-      {}
-    end
-
-    def ipv4_address
-      return @ipv4_address if defined?(@ipv4_address)
-
-      @ipv4_address = Resolv.getaddresses(@uri.host).find { |address| address =~ Resolv::IPv4::Regex }
-    rescue Resolv::ResolvError
-      @ipv4_address = nil
-    end
+      raise Error, "#{e.message}: start it with `npx @playwright/mcp --port 8931`" if e.message.include?("unreachable")
 
-    def next_id
-      @id = (@id || 0) + 1
+      raise
     end
   end
 end

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -45,21 +45,35 @@ module ActionAgent
         return run
       end
 
-      ensure_judge_defined_kpis! if @evaluation.judge_defined?
-
       records = scenarios.index_by(&:key)
-      report = Evals::Runner.new(
-        scenarios: scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) },
-        models: specs,
-        criteria: sample_criteria,
-        judge: evals_judge,
-        available_tools: tool_roster,
-        instructions: @evaluation.agent.instructions,
-        agent_name: @evaluation.agent.name,
-        threshold: PASS_THRESHOLD,
-        replay: ->(scenario, spec) { replay(scenario, spec) },
-        on_result: ->(result) { persist(run, records.fetch(result.scenario.key), result) }
-      ).call
+      tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
+      expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
+      # Every result tests membership twice, so look the pairs up in a set;
+      # `expected` stays an array for the completeness comparison below, and
+      # `persisted` sorts the same way either way.
+      allowed = expected.to_set
+      persisted = Set.new
+      on_result = lambda do |result|
+        pair = [ result.scenario.key, result.label ]
+        raise ArgumentError, "unexpected or duplicate scenario evaluation result" unless allowed.include?(pair) && !persisted.include?(pair)
+
+        persist(run, records.fetch(result.scenario.key), result)
+        persisted << pair
+      end
+      adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
+      report = if adapter
+        raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
+
+        adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
+      else
+        ensure_judge_defined_kpis! if @evaluation.judge_defined?
+        default_report(tasks, specs, on_result)
+      end
+      raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
+      reported = report.results.map { |result| [ result.scenario.key, result.label ] }
+      unless reported.sort == expected.sort && persisted.sort == expected.sort
+        raise ArgumentError, "scenario evaluation adapter must report and persist every selected scenario and model"
+      end
 
       run.update!(
         status: :complete,
@@ -76,6 +90,21 @@ module ActionAgent
 
     private
 
+    def default_report(tasks, specs, on_result)
+      Evals::Runner.new(
+        scenarios: tasks,
+        models: specs,
+        criteria: sample_criteria,
+        judge: evals_judge,
+        available_tools: tool_roster,
+        instructions: @evaluation.agent.instructions,
+        agent_name: @evaluation.agent.name,
+        threshold: PASS_THRESHOLD,
+        replay: ->(scenario, spec) { replay(scenario, spec) },
+        on_result: on_result
+      ).call
+    end
+
     # --- selection --------------------------------------------------------
 
     def selected_scenarios
@@ -184,7 +213,10 @@ module ActionAgent
         cost: result.replay.cost,
         fault: result.fault,
         recommendation: result.recommendation,
-        diagnosis: result.diagnosis || {},
+        diagnosis: (result.diagnosis || {}).merge(
+          "_replay_metadata" => result.replay.metadata,
+          "_scenario_snapshot" => result.scenario.to_h.merge(expectations: result.scenario.expectations)
+        ),
         error_message: result.replay.error
       )
     end
@@ -195,6 +227,8 @@ module ActionAgent
       scores["_recommendations"] = report.recommendations
       scores["_verdict"] = report.verdict if report.comparing?
       scores["_selection"] = run.selection
+      scores["_metadata"] = report.metadata
+      scores["_judge_label"] = report.judge_label || report.judge&.label
       scores
     end
 

data/config/routes.rb

@@ -20,6 +20,8 @@ ActionAgent::Engine.routes.draw do
 
   # The dashboard's own JSON API, read and written by the React app.
   namespace :api do
+    resource :dashboard_assistant, only: [ :show, :create ], controller: "dashboard_assistant"
+
     # Telemetry ingestion, relative to wherever the engine is mounted:
     # <mount>/api/traces (e.g. /activeagents/api/traces at the default mount).
     # Authenticated with a bearer token, not a session.
@@ -36,6 +38,10 @@ ActionAgent::Engine.routes.draw do
         post :duplicate
         get :export
         get :analytics
+        # The runner's conversation picker: this agent's persisted contexts,
+        # and a fresh one to pin a first message to.
+        get :conversations
+        post :conversations, action: :create_conversation
       end
       collection do
         get :presets
@@ -111,7 +117,11 @@ ActionAgent::Engine.routes.draw do
     resource :metrics, only: [ :show ], controller: "metrics"
 
     # Conversations (contexts, messages, generations) behind Interactions.
-    resources :interactions, only: [ :index, :show ]
+    # The runner edits a conversation in place — seeds, fixes or drops a
+    # turn — so the next run sees exactly the history it should.
+    resources :interactions, only: [ :index, :show ] do
+      resources :messages, only: [ :create, :update, :destroy ], controller: "interaction_messages"
+    end
 
     # Agent output evaluations. A scenario suite also manages its scenarios
     # here, and exposes each run's per-scenario, per-model results.

data/lib/action_agent/assistant_request_filter.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Runs before Rails' request logger, including for requests later rejected by
+  # authentication, consent or CSRF. Match the endpoint under any engine mount;
+  # leave the host application's unrelated message/history parameters alone.
+  class AssistantRequestFilter
+    PATH = %r{/api/dashboard_assistant(?:\.[^/]+)?/?\z}
+    PARAMETERS = [ /\Amessage\z/i, /\Ahistory\z/i ].freeze
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      if PATH.match?(env["PATH_INFO"].to_s)
+        env["action_dispatch.parameter_filter"] = Array(env["action_dispatch.parameter_filter"]) + PARAMETERS
+      end
+      @app.call(env)
+    end
+  end
+end

data/lib/action_agent/engine.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "assistant_request_filter"
+
 module ActionAgent
   # Rails engine for the Active Agent dashboard: the agent builder, runs,
   # conversations, evaluations, traces, metrics, sandboxes and session
@@ -19,6 +21,8 @@ module ActionAgent
     # rather than relying on the host to register one.
     INFLECTION_OVERRIDES = {
       "mcp_catalog" => "MCPCatalog",
+      "mcp_client" => "MCPClient",
+      "mcp_tool_dispatcher" => "MCPToolDispatcher",
       "mcp_controller" => "MCPController",
       "mcp_recording_middleware" => "MCPRecordingMiddleware",
       "mcp_servers_controller" => "MCPServersController",
@@ -35,6 +39,7 @@ module ActionAgent
     }.freeze
 
     config.action_agent = ActiveSupport::OrderedOptions.new
+    config.app_middleware.insert_before Rails::Rack::Logger, ActionAgent::AssistantRequestFilter
 
     # Whether a request is a browser asking for a page, as opposed to an API
     # or MCP client: the routes use it to tell the dashboard's client-side

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.3.0"
+  VERSION = "1.5.0"
 end

data/lib/action_agent.rb

@@ -236,6 +236,25 @@ module ActionAgent
     # @return [Boolean]
     attr_accessor :execution_enabled
 
+    # Whether the "Ask ActiveAgents" assistant is available.
+    #
+    # The assistant is a tool for developing and CI-ing agents: it sends
+    # recorded prompts, outputs and evaluation report excerpts to a model
+    # provider, which is the right trade in a development or CI workspace
+    # and a decision nobody should inherit by default in production. Left
+    # unset it is on in development and test only. Set it to true to run it
+    # somewhere else deliberately, or false to remove it everywhere.
+    # @return [Boolean, nil]
+    attr_accessor :assistant_enabled
+
+    # Resolves a host application's runner for one scenario evaluation.
+    # Return nil for the engine's normal Agent#test_execute path, or a callable
+    # accepting evaluation:, owner:, scenarios:, models:, on_result: and
+    # returning an ActiveAgent::Evals::Report. The host runs its own agent and
+    # judge and yields every result to on_result for dashboard persistence.
+    # @return [Proc, nil]
+    attr_accessor :scenario_evaluation_adapter_resolver
+
     # Where the dashboard's upgrade CTAs should send people. Unset in a
     # self-hosted install, where there is nothing to upgrade, and the CTAs
     # say so instead of linking nowhere.
@@ -332,6 +351,16 @@ module ActionAgent
       @execution_enabled != false
     end
 
+    # Returns whether the dashboard assistant is available. Unconfigured, it
+    # follows the environment: development and test yes, everywhere else no.
+    #
+    # @return [Boolean]
+    def assistant_enabled?
+      return @assistant_enabled == true unless @assistant_enabled.nil?
+
+      Rails.env.local?
+    end
+
     # Tells the host app that +owner+ performed +kind+. Never raises: a
     # bookkeeping failure must not fail the action that was already taken.
     def record_usage(owner, kind)
@@ -452,6 +481,9 @@ module ActionAgent
       @provider_credentials_resolver = nil
       @sandbox_backends = {}
       @execution_enabled = true
+      @assistant_enabled = nil
+
+      @scenario_evaluation_adapter_resolver = nil
       @table_name_prefix = "active_agent_"
       @agent_polymorphic_name = nil
       @encrypt_credentials = true

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -70,6 +70,18 @@ ActionAgent.configure do |config|
   # nothing rather than everything. An empty dashboard for a signed-in user
   # means the resolver above returned nil.
 
+  # ==========================================================================
+  # Ask ActiveAgents assistant
+  # ==========================================================================
+  #
+  # A tool for developing and CI-ing agents. Answering a question sends
+  # recorded prompts, outputs and evaluation report excerpts to a model
+  # provider, so the page and its API are on in development and test only.
+  # Set it to true to run it in another environment deliberately, or false
+  # to remove it everywhere.
+  #
+  # config.assistant_enabled = true
+
   # ==========================================================================
   # UI
   # ==========================================================================

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Justin Bowen
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -129,8 +128,10 @@ files:
 - app/controllers/action_agent/api/analytics_controller.rb
 - app/controllers/action_agent/api/api_keys_controller.rb
 - app/controllers/action_agent/api/base_controller.rb
+- app/controllers/action_agent/api/dashboard_assistant_controller.rb
 - app/controllers/action_agent/api/evaluations_controller.rb
 - app/controllers/action_agent/api/instance_tiers_controller.rb
+- app/controllers/action_agent/api/interaction_messages_controller.rb
 - app/controllers/action_agent/api/interactions_controller.rb
 - app/controllers/action_agent/api/mcp_controller.rb
 - app/controllers/action_agent/api/mcp_servers_controller.rb
@@ -193,10 +194,14 @@ files:
 - app/services/action_agent/agent_registrar.rb
 - app/services/action_agent/agent_scorecard.rb
 - app/services/action_agent/agent_toolbox.rb
+- app/services/action_agent/dashboard_assistant_service.rb
+- app/services/action_agent/evaluation_evidence.rb
 - app/services/action_agent/evaluation_runner_service.rb
 - app/services/action_agent/evaluation_tool_resolver.rb
 - app/services/action_agent/mcp_catalog.rb
+- app/services/action_agent/mcp_client.rb
 - app/services/action_agent/mcp_recording_middleware.rb
+- app/services/action_agent/mcp_tool_dispatcher.rb
 - app/services/action_agent/mock_sandbox_backend.rb
 - app/services/action_agent/playwright_mcp_client.rb
 - app/services/action_agent/sandbox_orchestrator.rb
@@ -212,6 +217,7 @@ files:
 - app/views/layouts/action_agent/react.html.erb
 - config/routes.rb
 - lib/action_agent.rb
+- lib/action_agent/assistant_request_filter.rb
 - lib/action_agent/compatibility.rb
 - lib/action_agent/engine.rb
 - lib/action_agent/version.rb
@@ -231,7 +237,6 @@ metadata:
   documentation_uri: https://docs.activeagents.ai/framework/self-hosted-observability
   source_code_uri: https://github.com/activeagents/activeagent
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -246,8 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.16
 specification_version: 4
 summary: The Active Agent dashboard, as a mountable Rails engine
 test_files: []