actionagent 1.9.5 → 1.9.7

data/app/services/action_agent/agent_scorecard.rb

@@ -101,16 +101,16 @@ module ActionAgent
       costs = Hash.new(0.0)
       priced = Set.new
 
-      run_models(windowed_runs).each do |agent_id, model, input, output|
-        cost = ModelPricing.estimate(model: model, input_tokens: input, output_tokens: output)
+      run_models(windowed_runs).each do |agent_id, model, provider, input, output|
+        cost = ModelPricing.estimate(model: model, provider: provider, input_tokens: input, output_tokens: output)
         next unless cost
 
         costs[agent_id] += cost
         priced << agent_id
       end
 
-      trace_models(unclaimed_traces(agent_ids, window_start)).each do |agent_id, model, input, output|
-        cost = ModelPricing.estimate(model: model, input_tokens: input, output_tokens: output)
+      trace_models(unclaimed_traces(agent_ids, window_start)).each do |agent_id, model, provider, input, output|
+        cost = ModelPricing.estimate(model: model, provider: provider, input_tokens: input, output_tokens: output)
         next unless cost
 
         costs[agent_id] += cost
@@ -123,27 +123,31 @@ module ActionAgent
     end
     private_class_method :estimated_costs
 
-    # [agent_id, model, input_tokens, output_tokens] per run. The model
-    # lives in the output_metadata JSON, which only PostgreSQL can reach
-    # from SQL; elsewhere the column is read back in Ruby.
+    # [agent_id, model, provider, input_tokens, output_tokens] per run. The
+    # model and the provider live in the output_metadata JSON, which only
+    # PostgreSQL can reach from SQL; elsewhere the column is read back in
+    # Ruby.
     def self.run_models(runs)
       if AgentRun.postgres?
-        runs.pluck(:agent_id, Arel.sql("output_metadata ->> 'model'"), :input_tokens, :output_tokens)
+        runs.pluck(:agent_id, Arel.sql("output_metadata ->> 'model'"), Arel.sql("output_metadata ->> 'provider'"),
+                   :input_tokens, :output_tokens)
       else
         runs.pluck(:agent_id, :output_metadata, :input_tokens, :output_tokens).map do |agent_id, metadata, input, output|
-          [ agent_id, metadata.is_a?(Hash) ? metadata["model"] : nil, input, output ]
+          metadata = {} unless metadata.is_a?(Hash)
+          [ agent_id, metadata["model"], metadata["provider"], input, output ]
         end
       end
     end
     private_class_method :run_models
 
-    # The same, for reported traces: the model is on the first llm span.
+    # The same, for reported traces: the model and provider are on the
+    # first llm span.
     def self.trace_models(traces)
       table = ActionAgent.trace_model.table_name
 
-      ActionAgent.trace_model.pluck_with_llm_model(
+      ActionAgent.trace_model.pluck_with_llm_model_and_provider(
         traces, Arel.sql("#{table}.agent_id"), :total_input_tokens, :total_output_tokens
-      ).map { |model, agent_id, input, output| [ agent_id, model, input, output ] }
+      ).map { |model, provider, agent_id, input, output| [ agent_id, model, provider, input, output ] }
     end
     private_class_method :trace_models
 

data/app/services/action_agent/agent_tool_roster.rb

@@ -26,8 +26,10 @@ module ActionAgent
   #   AgentToolbox turns into function schemas at generation time.
   # * **MCP** — never stored on the roster. Computed from the services the
   #   agent enables, which is where they are edited. The services are the
-  #   catalog plus the live checkout sandbox runtimes the caller hands in
-  #   (+runtimes:+), which an agent enables under their "sandbox:<id>" keys.
+  #   catalog plus the listings the caller hands in (+runtimes:+): the live
+  #   checkout sandbox runtimes, which an agent enables under their
+  #   "sandbox:<id>" keys, and the servers the owner added on the MCP Services
+  #   page, enabled under their "custom:<id>" keys.
   #
   # Enablement reads the agent's own configuration: +tools+ for the dashboard
   # capabilities and the schema tools, +mcp_servers+ for services and their
@@ -135,6 +137,7 @@ module ActionAgent
         first_party: catalog ? catalog[:first_party] : false,
         known: !catalog.nil?,
         runtime: SandboxSession.runtime_server_key?(key),
+        custom: MCPServerConfig.key?(key),
         transport: transport_label(catalog),
         status: service_status(calls, enabled, catalog),
         enabled: enabled,

data/app/services/action_agent/evaluation_fix_comparison.rb

@@ -21,7 +21,7 @@ module ActionAgent
       after = indexed(@session.verification_run)
       (before.keys | after.keys).map do |key|
         first, last = before[key], after[key]
-        { scenario_key: key[0], model: key[1], change: self.class.change(first, last),
+        { scenario_key: key[0], model: key[1], trial: key[2], change: self.class.change(first, last),
           before: first&.status, after: last&.status, before_score: first&.score&.to_f, after_score: last&.score&.to_f,
           score_change: first&.score && last&.score ? (last.score - first.score).to_f.round(3) : nil,
           before_fault: first&.fault, fault: last&.fault }
@@ -45,7 +45,8 @@ module ActionAgent
         "[Verification run](#{mount}/evaluations/#{base.evaluation_id}/runs/#{@session.verification_run_id})", "",
         "| Scenario / model | Before | After | Score change | Fault |", "| --- | --- | --- | --- | --- |" ]
       rows.each do |row|
-        cells = [ "#{row[:scenario_key]} / #{row[:model]}", row[:before], row[:after] || "pending", row[:score_change], row[:fault] || "—" ]
+        trial = " (trial #{row[:trial]})" if row[:trial] > 1
+        cells = [ "#{row[:scenario_key]} / #{row[:model]}#{trial}", row[:before], row[:after] || "pending", row[:score_change], row[:fault] || "—" ]
         lines << "| #{cells.map { |cell| cell.to_s.gsub('|', '\\|').gsub(/[\r\n]/, ' ') }.join(' | ')} |"
       end
       lines += [ "", "## Session summary", @session.result.to_s, "", "Code session ##{@session.id}. Review the diff and full-suite regressions before merging." ]
@@ -54,6 +55,9 @@ module ActionAgent
 
     private
 
+    # Results by scenario key, model and trial, so a run that replayed each
+    # scenario several times compares trial by trial rather than keeping
+    # whichever trial was read last.
     def indexed(run)
       return {} unless run
       item = @session.fix_item || {}
@@ -62,7 +66,7 @@ module ActionAgent
         model = [ result.provider, result.model ].compact_blank.join("/")
         next unless Array(item["scenario_keys"]).include?(key) &&
           (Array(item["models"]).include?(model) || Array(item["models"]).include?(result.model))
-        index[[ key, model ]] = result
+        index[[ key, model, result.trial ]] = result
       end
     end
   end

data/app/services/action_agent/evaluation_report_import.rb

@@ -18,7 +18,7 @@ module ActionAgent
   #                scope (SCOPE_KEYS in its metadata), named for both
   #   scenarios  — one per reported scenario key, updated to the reported prompt
   #   run        — one EvaluationRun per tenant and run_id, with one
-  #                EvaluationScenarioResult per scenario and model
+  #                EvaluationScenarioResult per scenario, model and trial
   #
   # An envelope's `status` is "complete" when absent. A "running" envelope
   # stores the results so far as a running run, and each later delivery under
@@ -71,6 +71,9 @@ module ActionAgent
 
     MAX_BYTES = 2.megabytes
     MAX_RESULTS = 1000
+    # The highest trial number the collector stores for a result (its
+    # `trial`, ActiveAgent::Evals::Result#trial).
+    MAX_TRIALS = 1000
     MAX_MODELS = 50
     MAX_TEXT = MAX_BYTES
     # Scenarios one evaluation may hold, and evaluations one agent may hold,
@@ -613,6 +616,7 @@ module ActionAgent
         recommendation: truncated(result["recommendation"], TEXT_BYTES),
         diagnosis: (result["diagnosis"] || {}).merge(
           "_replay_metadata" => result["metadata"] || {},
+          "_trial" => trial_of(result),
           "_scenario_snapshot" => {
             "key" => result["scenario_key"], "group" => result["group"], "prompt" => result["prompt"],
             "position" => scenario.position, "expectations" => {}
@@ -622,6 +626,12 @@ module ActionAgent
       )
     end
 
+    # Which replay of its scenario under its model a result is: 1 for a
+    # result that names none (an absent or null `trial`).
+    def trial_of(result)
+      result["trial"].nil? ? 1 : result["trial"]
+    end
+
     # A judge usage as the engine stores it: the keys the dashboard reads,
     # each cut to its type, or nil for none.
     def bounded_judge_usage(usage)
@@ -735,7 +745,9 @@ module ActionAgent
         object!(result, "result #{index}")
         %w[scenario_key label provider model].each { |key| string!(result[key], "result.#{key}", 200, required: true) }
         raise Invalid, "result label #{result['label']} is missing from report.models" unless report["models"].key?(result["label"])
-        raise Invalid, "duplicate scenario/model result" unless pairs.add?(result.values_at("scenario_key", "label"))
+        trial = trial_of(result)
+        raise Invalid, "result.trial must be an integer from 1 to #{MAX_TRIALS}" unless trial.is_a?(Integer) && trial.between?(1, MAX_TRIALS)
+        raise Invalid, "duplicate scenario/model result" unless pairs.add?([ result["scenario_key"], result["label"], trial ])
 
         spec = result.values_at("provider", "model")
         raise Invalid, "result label #{result['label']} names more than one provider/model" if label_specs.fetch(result["label"], spec) != spec

data/app/services/action_agent/evaluation_run_cost.rb

@@ -542,14 +542,14 @@ module ActionAgent
         return {} if scope.nil?
 
         table = ActionAgent.trace_model.table_name
-        rows = ActionAgent.trace_model.pluck_with_llm_model(
+        rows = ActionAgent.trace_model.pluck_with_llm_model_and_provider(
           scope.where(trace_id: ids),
           Arel.sql("#{table}.trace_id"), Arel.sql("#{table}.total_input_tokens"), Arel.sql("#{table}.total_output_tokens"),
           Arel.sql("#{table}.agent_action")
         )
-        rows.to_h do |model, trace_id, input, output, action|
+        rows.to_h do |model, provider, trace_id, input, output, action|
           [ trace_id.to_s, Trace.new(trace_id: trace_id.to_s, input_tokens: input.to_i, output_tokens: output.to_i,
-                                     model: model, provider: provider_of(model), action: action) ]
+                                     model: model, provider: provider.presence || provider_of(model), action: action) ]
         end
       rescue StandardError => e
         Rails.logger.warn("[ActionAgent] evaluation run #{@run.id} trace pricing failed: #{e.class}: #{e.message}")
@@ -565,8 +565,9 @@ module ActionAgent
 
       private
 
-      # A gateway's model names carry the vendor; the model's own name does
-      # not say which provider served it, which ModelPricing works around.
+      # For a trace whose llm span recorded no provider: a gateway's model
+      # names carry the vendor; the model's own name does not say which
+      # provider served it, which ModelPricing works around.
       def provider_of(model)
         head, rest = model.to_s.split("/", 2)
         rest.present? && ModelPricing::GATEWAY_PREFIXES.include?(head.downcase) ? head.downcase : nil

data/app/services/action_agent/evaluation_tool_resolver.rb

@@ -11,15 +11,18 @@ module ActionAgent
   # explicit +mcp__server__tool+ namespace wins because the name said so,
   # then MCPCatalog's hints for the bare names of well-known servers, then
   # the servers the agent itself declares in +mcp_servers+ when one of them
-  # lists the tool. Nothing here reads telemetry: the resolver runs inside
-  # the request that serializes a run, and every lookup is a constant or a
-  # single agent attribute.
+  # lists the tool, then the servers the agent's owner added on the MCP
+  # Services page (MCPServerConfig, "custom:<id>"), by the tools their last
+  # connection test listed. Nothing here reads telemetry: the resolver runs
+  # inside the request that serializes a run, and every lookup is a constant,
+  # a single agent attribute, or one query for the owner's configurations.
   #
   # The status is what the fix item's action turns on:
   #
   #   "enabled"   — the agent's mcp_servers configuration names the server
-  #   "available" — the catalog (built-in or ActionAgent.mcp_catalog) knows
-  #                 the server and the agent has not enabled it
+  #   "available" — the catalog (built-in or ActionAgent.mcp_catalog) or the
+  #                 owner's enabled configurations know the server, and the
+  #                 agent has not enabled it
   #   nil         — the namespace named a server nothing here knows; the
   #                 report renders that as "unknown"
   #
@@ -60,14 +63,15 @@ module ActionAgent
 
       ActiveAgent::Telemetry::ToolOrigin.server_for(name).presence ||
         MCPCatalog.server_for_tool(name).presence ||
-        configured_tools[name]
+        configured_tools[name] ||
+        custom_tools[name]
     end
 
     # @param key [String] a server key
     # @return [String, nil] ENABLED, AVAILABLE, or nil when unknown
     def status_for(key)
       return ENABLED if configured_keys.include?(normalize(key))
-      return AVAILABLE if MCPCatalog.find(key)
+      return AVAILABLE if MCPCatalog.find(key) || custom_config(key)
 
       nil
     end
@@ -105,6 +109,7 @@ module ActionAgent
     # (which is what MCPCatalog.display_name falls back to as well).
     def display_name_for(key)
       return MCPCatalog.display_name(key) if MCPCatalog.find(key)
+      return custom_config(key).name if custom_config(key)
 
       configured_names[normalize(key)] || MCPCatalog.display_name(key)
     end
@@ -145,6 +150,29 @@ module ActionAgent
       end
     end
 
+    # The agent's owner's enabled MCPServerConfig rows, read once. None
+    # without an agent: there is no owner to scope them to.
+    def custom_configs
+      @custom_configs ||= agent ? MCPServerConfig.for_owner(agent.try(:owner)).enabled.order(:id).to_a : []
+    end
+
+    def custom_config(key)
+      return nil unless MCPServerConfig.key?(key)
+
+      custom_configs.find { |config| config.server_key == key.to_s.strip }
+    end
+
+    # bare tool name => "custom:<id>", from the tools each of the owner's
+    # configurations listed when last tested. A configuration the agent
+    # enables claims a name before one it does not.
+    def custom_tools
+      @custom_tools ||= custom_configs
+        .sort_by { |config| [ configured_keys.include?(config.server_key) ? 0 : 1, config.id ] }
+        .each_with_object({}) do |config, map|
+          config.tool_names_list.each { |tool| map[tool] ||= config.server_key }
+        end
+    end
+
     # The agent's mcp_servers as a list of entries. Agents store an Array of
     # bare names or builder hashes, but an agent seeded from an older
     # template carries a top-level Hash keyed by server name

data/app/services/action_agent/mcp_client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "openssl"
 require "resolv"
 require "uri"
 require "json"
@@ -17,12 +18,24 @@ module ActionAgent
 
     class Error < StandardError; end
 
+    # Failures to reach the server at all, raised as Error so a caller
+    # handles one exception for a server that cannot be called.
+    NETWORK_ERRORS = [
+      Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
+      Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError, SocketError
+    ].freeze
+
     # +headers+ are sent with every request: a checkout sandbox's runtime
     # authenticates its MCP facade with a bearer token.
-    def initialize(url:, label: nil, headers: {})
+    #
+    # +public_only+ refuses a host that resolves to anything but public
+    # addresses (MCPHostGuard#public_addresses!), and connects to the address
+    # that was checked. A URL a dashboard user supplied is called this way.
+    def initialize(url:, label: nil, headers: {}, public_only: false)
       @uri = URI(url)
       @label = label.presence || @uri.host
       @headers = headers.to_h
+      @public_only = public_only
       @mutex = Mutex.new
     end
 
@@ -40,6 +53,8 @@ module ActionAgent
           parameters: tool["inputSchema"] || { type: "object", properties: {} }
         }
       end
+    rescue *NETWORK_ERRORS => e
+      raise Error, "MCP server #{@label} unreachable at #{@uri} (#{e.class})"
     end
 
     # Returns { text:, is_error: } — the tool result's text content.
@@ -63,7 +78,7 @@ module ActionAgent
 
       text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
       { text: text, is_error: result["isError"] ? true : false }
-    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
+    rescue *NETWORK_ERRORS => e
       raise Error, "MCP server #{@label} unreachable at #{@uri} (#{e.class})"
     end
 
@@ -101,6 +116,10 @@ module ActionAgent
       # fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
       # body comes back empty). A dedicated thread always does real blocking
       # IO outside any fiber/scheduler context.
+      #
+      # The address is resolved first, on the caller's thread, so a host the
+      # guard refuses raises here rather than inside the thread.
+      connect_address
       Thread.new { blocking_post_raw(payload, session: session) }.value
     end
 
@@ -111,8 +130,8 @@ module ActionAgent
       # Container->host bridge hostnames (host.orb.internal) publish an IPv6
       # address whose path doesn't reach the server; dual-stack connects then
       # fail intermittently. Pin to IPv4 while keeping the Host header.
-      if (ipv4 = ipv4_address)
-        http.ipaddr = ipv4
+      if (address = connect_address)
+        http.ipaddr = address
       end
       http.open_timeout = OPEN_TIMEOUT_SECONDS
       http.read_timeout = READ_TIMEOUT_SECONDS
@@ -161,6 +180,19 @@ module ActionAgent
       {}
     end
 
+    # The address to connect to. For a public-only client, a checked public
+    # address (IPv4 first), resolved once so every request of this client
+    # reaches the address that was checked; otherwise the host's IPv4
+    # address when it has one.
+    def connect_address
+      return ipv4_address unless @public_only
+
+      @checked_address ||= begin
+        addresses = MCPHostGuard.public_addresses!(@uri.hostname)
+        addresses.find { |address| address =~ Resolv::IPv4::Regex } || addresses.first
+      end
+    end
+
     def ipv4_address
       return @ipv4_address if defined?(@ipv4_address)
 

data/app/services/action_agent/mcp_host_guard.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+require "resolv"
+require "uri"
+
+module ActionAgent
+  # Decides whether the dashboard may call a user-supplied MCP server URL
+  # (MCPServerConfig). Those servers are reached from the dashboard's own
+  # network, so without a check any user could point one at an address only
+  # the dashboard can reach: a cloud metadata endpoint, a database admin page
+  # on the private network, the host's own loopback.
+  #
+  # The check runs twice:
+  #
+  # * When a configuration is saved, #url_refusal reads the URL alone, with
+  #   no DNS lookup: the scheme is http or https, the URL carries no user
+  #   name or password, and its host is not a literal non-public address or a
+  #   localhost name.
+  # * When MCPClient connects (+public_only: true+), #public_addresses!
+  #   resolves the host and refuses it unless every address it resolves to
+  #   is public. The client then connects to the address that was checked,
+  #   so a name whose DNS answer changes between the check and the connection
+  #   still reaches only a checked address.
+  #
+  # ActionAgent.allow_private_mcp_hosts switches both off, for local
+  # development against a server on localhost.
+  class MCPHostGuard
+    SCHEMES = %w[http https].freeze
+
+    # Addresses no public server is reached at, beyond what IPAddr already
+    # names (loopback?, private?, link_local?): the unspecified "this host"
+    # ranges, carrier-grade NAT, benchmarking, multicast and reserved space.
+    NON_PUBLIC_RANGES = %w[
+      0.0.0.0/8 100.64.0.0/10 192.0.0.0/24 198.18.0.0/15 224.0.0.0/4 240.0.0.0/4
+      ::/128 ff00::/8
+    ].map { |range| IPAddr.new(range) }.freeze
+
+    # Host names that resolve to the machine itself however DNS answers.
+    LOCAL_NAMES = /\A(?:localhost|.+\.localhost|localhost\.localdomain)\z/i
+
+    class << self
+      # Resolves a host name to its addresses. Replaceable so tests need no
+      # network.
+      # @return [#call]
+      attr_writer :resolver
+
+      def resolver
+        @resolver || Resolv.method(:getaddresses)
+      end
+
+      # Whether private addresses are allowed (ActionAgent.allow_private_mcp_hosts).
+      def allow_private?
+        ActionAgent.allow_private_mcp_hosts ? true : false
+      end
+
+      # Why +url+ cannot be saved as an MCP server URL, or nil when it can.
+      # Reads the URL alone and resolves nothing.
+      #
+      # @param url [String]
+      # @param allow_private [Boolean]
+      # @return [String, nil]
+      def url_refusal(url, allow_private: allow_private?)
+        uri = URI.parse(url.to_s.strip)
+        return "must be an http:// or https:// URL" unless SCHEMES.include?(uri.scheme.to_s.downcase)
+        return "must name a host" if uri.hostname.blank?
+        return "must not carry a user name or password; send credentials in a header instead" if uri.userinfo.present?
+        return nil if allow_private
+
+        host = uri.hostname
+        return "must not point at localhost" if LOCAL_NAMES.match?(host)
+
+        address = literal_address(host)
+        return "must not point at a private, loopback or link-local address" if address && !public_address?(address)
+
+        nil
+      rescue URI::InvalidURIError
+        "is not a valid URL"
+      end
+
+      # The addresses +host+ resolves to, every one of them public.
+      #
+      # @param host [String]
+      # @return [Array<String>]
+      # @raise [MCPClient::Error] when the host does not resolve, or resolves
+      #   to any non-public address
+      def public_addresses!(host)
+        addresses = Array(resolver.call(host.to_s)).map(&:to_s).uniq
+        raise MCPClient::Error, "#{host} does not resolve to an address" if addresses.empty?
+
+        refused = addresses.reject { |address| public_address?(address) }
+        if refused.any?
+          raise MCPClient::Error,
+                "#{host} resolves to a private, loopback or link-local address (#{refused.first}); " \
+                "set ActionAgent.allow_private_mcp_hosts to reach it"
+        end
+
+        addresses
+      rescue Resolv::ResolvError => e
+        raise MCPClient::Error, "#{host} does not resolve: #{e.message}"
+      end
+
+      # Whether +address+ (an IP address string or IPAddr) is publicly
+      # routable. An IPv4 address written as IPv6 (::ffff:127.0.0.1) is read
+      # as the IPv4 address it carries. Anything that does not parse is not
+      # public.
+      #
+      # @return [Boolean]
+      def public_address?(address)
+        ip = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
+        ip = ip.native
+        return false if ip.loopback? || ip.private? || ip.link_local?
+
+        NON_PUBLIC_RANGES.none? { |range| range.family == ip.family && range.include?(ip) }
+      rescue IPAddr::InvalidAddressError, IPAddr::AddressFamilyError
+        false
+      end
+
+      private
+
+      # +host+ as an IPAddr when it is an IP literal, nil when it is a name.
+      def literal_address(host)
+        IPAddr.new(host)
+      rescue IPAddr::InvalidAddressError, IPAddr::AddressFamilyError
+        nil
+      end
+    end
+  end
+end

data/app/services/action_agent/mcp_tool_dispatcher.rb

@@ -23,6 +23,12 @@ module ActionAgent
   # is reached the same way (SandboxSession.browser_server_entry), and only
   # this way. Only those keys are taken; a catalog server the agent does not
   # enable stays out of reach.
+  #
+  # A server a dashboard user added on the MCP Services page is enabled as
+  # "custom:<id>" and resolves through MCPServerConfig.server_entry, among the
+  # agent's owner's configurations only and only while it is enabled. Its
+  # client refuses a host that resolves to a non-public address (see
+  # MCPHostGuard) unless the host app allows them.
   class MCPToolDispatcher
     HTTP_TRANSPORTS = %w[http streamable_http sse].freeze
     class SandboxUnavailable < MCPClient::Error; end
@@ -43,6 +49,7 @@ module ActionAgent
     def self.unreachable_reason(entry)
       return "Unknown to the catalog" if entry.nil?
       return nil if SandboxSession.runtime_server_key?(entry[:key].to_s)
+      return "Switched off on the MCP Services page" if entry[:custom] && entry[:enabled] == false
 
       transport = entry[:transport].to_s
       unless transport.in?(HTTP_TRANSPORTS)
@@ -284,14 +291,16 @@ module ActionAgent
     end
 
     # A catalog server, the app runtime of a checkout sandbox the agent's
-    # owner started (keys "sandbox:<session_id>"), or, for a key this run was
-    # given, that sandbox's browser ("browser:<session_id>"). The sandbox
-    # lookups are scoped to the agent's owner, so naming another tenant's
-    # session resolves to nothing.
+    # owner started (keys "sandbox:<session_id>"), a server the agent's owner
+    # added ("custom:<id>"), or, for a key this run was given, that sandbox's
+    # browser ("browser:<session_id>"). The sandbox and custom lookups are
+    # scoped to the agent's owner, so naming another tenant's session or
+    # configuration resolves to nothing.
     def catalog_entry(key)
       if SandboxSession.browser_server_key?(key)
         return extra_server_keys.include?(key) ? SandboxSession.browser_server_entry(key, owner: agent.try(:owner)) : nil
       end
+      return MCPServerConfig.server_entry(key, owner: agent.try(:owner)) if MCPServerConfig.key?(key)
       return MCPCatalog.connection(key) unless SandboxSession.runtime_server_key?(key)
 
       SandboxSession.runtime_server_entry(key, owner: agent.try(:owner))
@@ -303,7 +312,8 @@ module ActionAgent
       @clients[entry[:key]] ||= MCPClient.new(
         url: absolute_url(entry[:url]),
         label: entry[:name] || entry[:key],
-        headers: entry[:headers] || {}
+        headers: entry[:headers] || {},
+        public_only: entry[:public_only] ? true : false
       )
     end
 

data/app/services/action_agent/ollama_host_probe.rb

@@ -10,12 +10,19 @@ module ActionAgent
   # The host is the OpenAI-compatible base URL (".../v1"); the probe calls
   # GET {host}/models. An optional API key is sent as a Bearer token for
   # remote servers behind an authenticating proxy or Ollama Cloud.
+  #
+  # The result says whether the host is Ollama Cloud (`cloud`) and, for
+  # each listed model a price list names, its rate (`rates`): the cloud's
+  # models are metered per token, and a local server's copy of a cloud
+  # model is priced at the same list rate (ModelPricing). A model priced
+  # only by a guess has no entry.
   class OllamaHostProbe
     TIMEOUT_SECONDS = 4
+    CLOUD_HOST = "ollama.com"
 
-    Result = Struct.new(:ok, :host, :models, :latency_ms, :error, keyword_init: true) do
+    Result = Struct.new(:ok, :host, :models, :latency_ms, :error, :cloud, :rates, keyword_init: true) do
       def to_h
-        { ok: ok, host: host, models: models, latency_ms: latency_ms, error: error }
+        { ok: ok, host: host, models: models, latency_ms: latency_ms, error: error, cloud: cloud, rates: rates }
       end
     end
 
@@ -23,11 +30,24 @@ module ActionAgent
       new(host: host, api_key: api_key).call
     end
 
+    # Whether +host+ addresses Ollama Cloud (the framework's
+    # ActiveAgent::Providers::Ollama::Cloud.cloud_host?, kept in step).
+    def self.cloud_host?(host)
+      name = URI.parse(host.to_s.strip).host.to_s.downcase
+      name == CLOUD_HOST || name.end_with?(".#{CLOUD_HOST}")
+    rescue URI::InvalidURIError
+      false
+    end
+
     def initialize(host:, api_key: nil)
       @host = ProviderKey.normalize_host(host)
       @api_key = api_key.presence
     end
 
+    def cloud?
+      self.class.cloud_host?(@host)
+    end
+
     def call
       return failure("No host configured") if @host.blank?
 
@@ -49,7 +69,7 @@ module ActionAgent
 
       data = JSON.parse(response.body)
       models = Array(data["data"]).filter_map { |model| model["id"] }.sort
-      Result.new(ok: true, host: @host, models: models, latency_ms: latency_ms, error: nil)
+      Result.new(ok: true, host: @host, models: models, latency_ms: latency_ms, error: nil, cloud: cloud?, rates: rates_for(models))
     rescue JSON::ParserError
       failure("#{@host} did not return JSON — is this the OpenAI-compatible /v1 endpoint?")
     rescue URI::InvalidURIError, ArgumentError
@@ -68,8 +88,20 @@ module ActionAgent
       @api_key ? { "Authorization" => "Bearer #{@api_key}" } : {}
     end
 
+    # `{ "gpt-oss:120b" => { input: 0.15, output: 0.6, source: "catalog" } }`
+    # for every listed model a price list names, in $ per million tokens.
+    def rates_for(models)
+      provider = cloud? ? ModelPricing::OLLAMA_CLOUD_PROVIDER : "ollama"
+      models.each_with_object({}) do |id, rates|
+        detail = ModelPricing.rate_detail(id, provider: provider)
+        next if ModelPricing::FALLBACK_SOURCES.include?(detail[:source])
+
+        rates[id] = detail.slice(:input, :output, :source)
+      end
+    end
+
     def failure(message, latency_ms = nil)
-      Result.new(ok: false, host: @host, models: [], latency_ms: latency_ms, error: message)
+      Result.new(ok: false, host: @host, models: [], latency_ms: latency_ms, error: message, cloud: cloud?, rates: {})
     end
   end
 end

data/app/services/action_agent/scenario_catalog_import.rb

@@ -213,6 +213,8 @@ module ActionAgent
           prompt: scenario.prompt, notes: scenario.notes, expectations: scenario.expectations,
           tags: scenario.tags, params: scenario.params, production_only: scenario.production_only?
         )
+        # Until add_catalog_scenario_metadata runs, imports keep working without extensions.
+        record.metadata = scenario.metadata if record.has_attribute?(:metadata)
         record.position = position_for(record, scenario.position, offset)
         record.save!
       end