actionagent 1.9.5 → 1.9.7

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -41,6 +41,7 @@ module ActionAgent
   # any replay, and one started here is stopped once the run ends.
   class ScenarioEvaluationRunner < EvaluationRunnerService
     Evals = ActiveAgent::Evals
+    JUDGE_LEASE_DURATION = 30.minutes
 
     # The providers a candidate model name resolves against. `mock` is the
     # framework's test double, accepted so the test suite can compare cohorts
@@ -187,17 +188,20 @@ module ActionAgent
     # A job delivered again for a result that is already scored, or that a
     # concurrent delivery has claimed, scores nothing. It still checks
     # whether the run is complete: the earlier delivery may have stored the
-    # last result and stopped before completing the run.
+    # last result and stopped before completing the run. A result another
+    # delivery holds a live lease on is tried again when the lease expires
+    # (see #wait_for_claim!).
     def judge_result!(row)
       run = row.evaluation_run
       return unless run.running?
-      return finish_fan_out!(run) unless row.pending? && claim_result!(row)
+      return finish_fan_out!(run) unless row.pending?
+      return wait_for_claim!(row, run) unless claim_result!(row)
 
       spec = fan_out_specs(run).fetch([ row.provider.to_s, row.model ]) do
         Evals::ModelSpec.new(label: row.model, provider: row.provider.to_s, model: row.model)
       end
       result = fan_out_scorer(run).evaluate(Evals::Scenario.from_hash(row.evaluated_scenario), spec, replay_from(row))
-      row.update!(
+      store_claimed_result!(row,
         status: result.status,
         score: result.score,
         scores: result.scores,
@@ -227,7 +231,15 @@ module ActionAgent
           row = stored_result(run, scenario, spec)
           task = Evals::Scenario.from_hash(scenario.as_json_summary)
           row ||= store_replay(run, scenario, task, spec, Evals::Replay.failed(error))
-          row.update!(status: :errored, error_message: row.error_message.presence || error_text(error)) if row.pending?
+          attributes = { status: :errored, error_message: row.error_message.presence || error_text(error) }
+          if scoring_leases?
+            # Another replay delivery may fail while this result is being
+            # scored. Only an unclaimed replay belongs to this failure path.
+            EvaluationScenarioResult.where(id: row.id, status: :pending, scoring_token: nil)
+              .update_all(attributes.merge(updated_at: Time.current))
+          elsif row.pending?
+            row.update!(attributes)
+          end
         end
       end
       finish_fan_out!(run)
@@ -237,7 +249,7 @@ module ActionAgent
     # still completes.
     def fail_judging!(row, error)
       run = row.evaluation_run
-      row.update!(status: :errored, error_message: [ row.error_message.presence, "Scoring failed: #{error_text(error)}" ].compact.join(" ")) if row.pending?
+      store_claimed_result!(row, status: :errored, error_message: [ row.error_message.presence, "Scoring failed: #{error_text(error)}" ].compact.join(" ")) if row.pending?
       finish_fan_out!(run)
     end
 
@@ -331,15 +343,59 @@ module ActionAgent
       @evaluation.scenarios.where(id: ids).count * specs.size
     end
 
-    # Claims a pending result for this job before it is scored. Of two judge
-    # jobs delivered for one result (a replay job delivered again after it
-    # enqueued the first), only the one whose claim lands scores it, so the
-    # judge is asked once and the run's meter counts each call once. The
-    # claim is a conditional write on the row's timestamp: it holds no lock
-    # while the judge is asked, and a row whose claimant then stops is still
-    # pending for the next delivery.
+    # A timestamp comparison alone only rejects a stale reader. A duplicate
+    # delivered after that write reads the new timestamp and could score too.
+    # The lease excludes fresh readers as well, without holding a database
+    # lock across a provider call. A redelivery can reclaim an expired lease.
+    #
+    # Without the lease columns (migration 030 not yet run) the claim is the
+    # timestamp comparison alone, as before the lease.
     def claim_result!(row)
-      EvaluationScenarioResult.where(id: row.id, status: :pending, updated_at: row.updated_at).update_all(updated_at: Time.current) == 1
+      now = Time.current
+      claim = EvaluationScenarioResult.where(id: row.id, status: :pending, updated_at: row.updated_at)
+      return claim.update_all(updated_at: now) == 1 unless scoring_leases?
+
+      token = SecureRandom.uuid
+      claimed = claim.where("scoring_token IS NULL OR scoring_expires_at <= ?", now)
+        .update_all(scoring_token: token, scoring_expires_at: now + JUDGE_LEASE_DURATION, updated_at: now)
+      @scoring_token = token if claimed == 1
+    end
+
+    # A claim lost to another delivery leaves the result pending, and that
+    # delivery may be a worker that stopped: the queue's redelivery of its
+    # job lands here, inside the lease the stopped worker still holds. So the
+    # job is enqueued again for when that lease expires. If the holder stores
+    # the result first, the later delivery finds it scored and only checks
+    # whether the run is complete. A result left pending with no live lease
+    # (the claim raced another write to the row) is tried again at once.
+    # Without the lease columns a lost claim is a concurrent delivery that is
+    # scoring the result now, as before.
+    def wait_for_claim!(row, run)
+      current = EvaluationScenarioResult.find_by(id: row.id) if scoring_leases?
+      if current&.pending?
+        retry_at = [ current.scoring_expires_at, Time.current ].compact.max
+        EvaluationJudgeJob.set(wait_until: retry_at).perform_later(row.id, @selection.to_h, run.id)
+      end
+      finish_fan_out!(run)
+    end
+
+    # Fence late successes and failures: after a lease is reclaimed, the old
+    # worker cannot overwrite its successor. Only the current owner releases
+    # the lease. A failure before claiming may mark an unclaimed row errored.
+    def store_claimed_result!(row, **attributes)
+      return row.update!(attributes) unless scoring_leases?
+
+      row.assign_attributes(attributes)
+      row.validate!
+      EvaluationScenarioResult.where(id: row.id, status: :pending, scoring_token: @scoring_token)
+        .update_all(attributes.merge(scoring_token: nil, scoring_expires_at: nil, updated_at: Time.current))
+    end
+
+    # Whether migration 030's lease columns are installed. A host that
+    # upgraded the gem without running the generator keeps scoring with the
+    # timestamp claim instead of failing every judge job on a missing column.
+    def scoring_leases?
+      (%w[scoring_token scoring_expires_at] - EvaluationScenarioResult.column_names).empty?
     end
 
     # What Evals::Runner does with a replay that raises: an errored replay,

data/app/services/action_agent/tool_discovery.rb

@@ -37,7 +37,9 @@ module ActionAgent
   # rules (single-user, per-user, or multi-tenant) decide what is visible.
   # The same goes for checkout sandbox runtimes: the caller hands in the live
   # ones it may see (SandboxSession.runtime_server_listings), and they are
-  # listed beside the catalog under their "sandbox:<session_id>" keys.
+  # listed beside the catalog under their "sandbox:<session_id>" keys. The
+  # servers the owner added (MCPServerConfig.listings) are handed in the same
+  # way, under their "custom:<id>" keys.
   class ToolDiscovery
     DEFAULT_WINDOW_HOURS = 24 * 7
     MAX_WINDOW_HOURS = 24 * 90
@@ -543,6 +545,8 @@ module ActionAgent
         # from traffic is real but undocumented here, and the view says so.
         known: !catalog.nil?,
         runtime: runtime,
+        # A server the owner added on the MCP Services page ("custom:<id>").
+        custom: MCPServerConfig.key?(key),
         status: server_status(calls, configured, catalog),
         calls: calls,
         errors: bucket ? bucket[:errors] : 0,

data/config/routes.rb

@@ -193,6 +193,12 @@ ActionAgent::Engine.routes.draw do
       end
     end
 
+    # The caller's own remote MCP servers, enabled on agents as
+    # "custom:<id>". Header values are write-only; test runs tools/list.
+    resources :mcp_server_configs, only: [ :index, :create, :update, :destroy ] do
+      post :test, on: :member
+    end
+
     resources :instance_tiers, only: [ :index, :show ] do
       collection do
         get :recommend

data/lib/action_agent/engine.rb

@@ -1,5 +1,12 @@
 # frozen_string_literal: true
 
+# The class body below reads Zeitwerk's inflector. Rails requires Zeitwerk
+# only when an application sets up its autoloaders, which is after
+# Bundler.require has loaded this file; a host that reaches it earlier does so
+# by chance (Action Cable's engine loads it as a side effect). So the constant
+# has to come from here. Zeitwerk is a dependency of Active Support, so it is
+# always installed.
+require "zeitwerk"
 require_relative "assistant_request_filter"
 
 module ActionAgent
@@ -25,7 +32,10 @@ module ActionAgent
       "mcp_tool_dispatcher" => "MCPToolDispatcher",
       "mcp_controller" => "MCPController",
       "mcp_dashboard_tools" => "MCPDashboardTools",
+      "mcp_host_guard" => "MCPHostGuard",
       "mcp_recording_middleware" => "MCPRecordingMiddleware",
+      "mcp_server_config" => "MCPServerConfig",
+      "mcp_server_configs_controller" => "MCPServerConfigsController",
       "mcp_servers_controller" => "MCPServersController",
       "playwright_mcp_client" => "PlaywrightMCPClient"
     }.freeze

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.9.5"
+  VERSION = "1.9.7"
 end

data/lib/action_agent.rb

@@ -887,6 +887,22 @@ module ActionAgent
     # @return [Array<Hash>]
     attr_accessor :mcp_catalog
 
+    # Whether a dashboard user's own MCP server (MCPServerConfig, added on the
+    # MCP Services page) may point at a loopback, private, link-local or
+    # otherwise non-public address. Off by default: those servers are reached
+    # from the dashboard's own network, so a URL such as
+    # http://169.254.169.254/ or http://10.0.0.5/admin would let any user make
+    # the dashboard call an address only it can reach. MCPHostGuard refuses
+    # such a URL when it is saved, and refuses a host name that resolves to
+    # one when it is called. Turn it on for local development, where the
+    # server under test runs on localhost, or for a single-user install whose
+    # user already has that network access.
+    #
+    # Host registrations (+mcp_catalog+) and checkout sandbox runtimes are the
+    # host app's own configuration and are not affected.
+    # @return [Boolean]
+    attr_accessor :allow_private_mcp_hosts
+
     # Host-declared ActiveAgent::SchemaTools subclasses whose generated tools
     # are offered alongside AgentToolbox's built-ins: each generated tool is
     # individually selectable in the agent editor, dispatched by name at
@@ -1358,6 +1374,7 @@ module ActionAgent
       @user_menu_links = nil
       @sign_in_path = nil
       @mcp_catalog = []
+      @allow_private_mcp_hosts = false
       @agent_actor_resolver = nil
       @schema_tools = nil
       @schema_tools_path = "app/agent_tools"

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -317,6 +317,17 @@ ActionAgent.configure do |config|
   # Expired sandboxes are stopped by `bin/rails action_agent:sandbox:reap`.
   # Schedule it every few minutes.
 
+  # ==========================================================================
+  # MCP servers
+  # ==========================================================================
+  #
+  # Users add their own remote MCP servers on the MCP Services page. Their
+  # URLs may not reach a loopback, private or link-local address unless this
+  # is on: those servers are called from the dashboard's own network. Turn it
+  # on in development to test a server running on localhost.
+  #
+  # config.allow_private_mcp_hosts = Rails.env.development?
+
   # ==========================================================================
   # UI
   # ==========================================================================

data/lib/generators/action_agent/templates/migrations/030_add_evaluation_scoring_leases.rb.erb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddEvaluationScoringLeases < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column "#{ActionAgent.table_name_prefix}evaluation_scenario_results", :scoring_token, :string
+    add_column "#{ActionAgent.table_name_prefix}evaluation_scenario_results", :scoring_expires_at, :datetime
+  end
+end

data/lib/generators/action_agent/templates/migrations/031_add_catalog_scenario_metadata.rb.erb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AddCatalogScenarioMetadata < ActiveRecord::Migration<%= migration_version %>
+  def up
+    return unless table_exists?(scenarios_table)
+    return if column_exists?(scenarios_table, :metadata)
+
+    type = connection.adapter_name.to_s.downcase.include?("postgres") ? :jsonb : :json
+    add_column scenarios_table, :metadata, type
+  end
+
+  def down
+    return unless table_exists?(scenarios_table)
+    return unless column_exists?(scenarios_table, :metadata)
+
+    remove_column scenarios_table, :metadata
+  end
+
+  private
+
+  def scenarios_table
+    "#{ActionAgent.table_name_prefix}catalog_scenarios"
+  end
+end

data/lib/generators/action_agent/templates/migrations/032_create_active_agent_mcp_server_configs.rb.erb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# MCP servers a dashboard user adds on the MCP Services page
+# (ActionAgent::MCPServerConfig): a remote server's URL, its request headers
+# (encrypted at rest, in secret_headers), and what the last connection test
+# found. Each row belongs to an account or a user, as provider keys do.
+#
+# Table names follow ActionAgent.table_name_prefix, and JSON columns are
+# jsonb on PostgreSQL and json elsewhere, the same way as
+# create_active_agent_dashboard_tables.
+class CreateActiveAgentMCPServerConfigs < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}mcp_server_configs" do |t|
+      t.string :name, null: false
+      t.text :description
+      t.string :url, null: false, limit: 2048
+      # A JSON object of header name => value, encrypted.
+      t.text :secret_headers
+      t.boolean :enabled, null: false, default: true
+      t.datetime :last_tested_at
+      # "ok" or "failed", and the error of a failed test.
+      t.string :last_test_status
+      t.text :last_test_error
+      # The tool names the last successful test listed.
+      t.column :tool_names, json_type, **json_default([])
+      t.bigint :account_id
+      t.bigint :user_id
+      t.timestamps
+      t.index :account_id, name: "index_#{prefix}mcp_server_configs_on_account"
+      t.index :user_id, name: "index_#{prefix}mcp_server_configs_on_user"
+    end
+  end
+
+  private
+
+  def json_type
+    @json_type ||= postgres? ? :jsonb : :json
+  end
+
+  # MySQL rejects a default on a JSON column outright, so the column is
+  # created without one there.
+  def json_default(value, null: nil)
+    return {} unless postgres?
+
+    null.nil? ? { default: value } : { default: value, null: null }
+  end
+
+  def postgres?
+    connection.adapter_name.to_s.downcase.include?("postgres")
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.9.5
+  version: 1.9.7
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-28 00:00:00.000000000 Z
+date: 2026-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -150,6 +150,7 @@ files:
 - app/controllers/action_agent/api/interaction_messages_controller.rb
 - app/controllers/action_agent/api/interactions_controller.rb
 - app/controllers/action_agent/api/mcp_controller.rb
+- app/controllers/action_agent/api/mcp_server_configs_controller.rb
 - app/controllers/action_agent/api/mcp_servers_controller.rb
 - app/controllers/action_agent/api/members_controller.rb
 - app/controllers/action_agent/api/metrics_controller.rb
@@ -229,6 +230,7 @@ files:
 - app/models/action_agent/github_webhook_delivery.rb
 - app/models/action_agent/input_request.rb
 - app/models/action_agent/interaction_feedback.rb
+- app/models/action_agent/mcp_server_config.rb
 - app/models/action_agent/model_pricing.rb
 - app/models/action_agent/project.rb
 - app/models/action_agent/project_secret.rb
@@ -297,6 +299,7 @@ files:
 - app/services/action_agent/local_sandbox_databases.rb
 - app/services/action_agent/mcp_catalog.rb
 - app/services/action_agent/mcp_client.rb
+- app/services/action_agent/mcp_host_guard.rb
 - app/services/action_agent/mcp_recording_middleware.rb
 - app/services/action_agent/mcp_tool_dispatcher.rb
 - app/services/action_agent/mock_sandbox_backend.rb
@@ -377,6 +380,9 @@ files:
 - lib/generators/action_agent/templates/migrations/027_create_active_agent_interaction_feedbacks.rb.erb
 - lib/generators/action_agent/templates/migrations/028_add_conversations_to_code_sessions.rb.erb
 - lib/generators/action_agent/templates/migrations/029_create_active_agent_github_webhook_deliveries.rb.erb
+- lib/generators/action_agent/templates/migrations/030_add_evaluation_scoring_leases.rb.erb
+- lib/generators/action_agent/templates/migrations/031_add_catalog_scenario_metadata.rb.erb
+- lib/generators/action_agent/templates/migrations/032_create_active_agent_mcp_server_configs.rb.erb
 - lib/tasks/action_agent.rake
 - lib/tasks/browser.rake
 - lib/tasks/claude_code.rake