actionagent 1.2.2 → 1.3.0

data/config/routes.rb

@@ -70,7 +70,7 @@ ActionAgent::Engine.routes.draw do
     resources :tools, only: [ :index ]
 
     # MCP services — detected servers unioned with the default catalog
-    # (McpCatalog), plus on-demand sandbox provisioning. Keys are catalog
+    # (MCPCatalog), plus on-demand sandbox provisioning. Keys are catalog
     # slugs like "sequential-thinking", so the id segment allows dashes.
     resources :mcp_servers, only: [ :index, :show ], id: /[^\/]+/ do
       member do
@@ -113,10 +113,17 @@ ActionAgent::Engine.routes.draw do
     # Conversations (contexts, messages, generations) behind Interactions.
     resources :interactions, only: [ :index, :show ]
 
-    # Agent output evaluations.
+    # Agent output evaluations. A scenario suite also manages its scenarios
+    # here, and exposes each run's per-scenario, per-model results.
     resources :evaluations, only: [ :index, :show, :create, :destroy ] do
       member do
         post :run
+        get "runs/:run_id", action: :show_run, as: :run_result
+        get "runs/:run_id/report", action: :run_report, as: :run_report
+        get :scenarios
+        put :scenarios, action: :replace_scenarios
+        patch "scenarios/:scenario_id", action: :update_scenario, as: :scenario
+        delete "scenarios/:scenario_id", action: :destroy_scenario
       end
     end
 
@@ -128,6 +135,12 @@ ActionAgent::Engine.routes.draw do
     # Model catalogs for the agent builder (Ollama queried live from the
     # configured host; hosted providers curated).
     resources :provider_models, only: [ :index ]
+
+    # The plan meter the Organization view and the Run Agents quota banner
+    # read. The engine meters nothing itself: a host that tracks usage
+    # against a plan answers through ActionAgent.usage_resolver, and a bare
+    # mount reports unlimited rather than 404.
+    resource :usage, only: [ :show ], controller: "usage"
   end
 
   # The account's agents presented as an authenticated MCP server (tools +
@@ -136,6 +149,16 @@ ActionAgent::Engine.routes.draw do
   # namespace's session-authenticated controllers.
   post "mcp", to: "api/mcp#create"
 
+  # MCP Streamable HTTP (2025-03-26): a client MAY open the server-to-client
+  # SSE stream with GET, and ends a session with DELETE. This facade offers
+  # no stream and keeps no sessions, so both answer 405 with Allow: POST —
+  # the clean "not offered" signal SDK clients expect, instead of the
+  # dashboard's HTML page parsed as an event stream. A browser's GET (Accept
+  # prefers HTML) is the MCP Services view's deep link, and falls through to
+  # the catch-all below like any other client-side route.
+  match "mcp", to: "api/mcp#unsupported", via: [ :get, :delete ],
+    constraints: ->(request) { request.delete? || !ActionAgent::Engine.html_request?(request) }
+
   # Everything else under the mount is a client-side route: render the
   # dashboard and let the browser resolve it. Anchored last so it can only
   # ever catch what the routes above did not, and refuses /api paths so a

data/lib/action_agent/engine.rb

@@ -14,14 +14,37 @@ module ActionAgent
     engine_name "action_agent"
 
     # Basenames this engine spells differently from Zeitwerk's default
-    # camelization, consulted by the inflections initializer below. Empty
-    # today: every file here is named for the constant default camelization
-    # produces. An engine file that wants a genuine acronym in its constant
-    # adds its basename here rather than relying on the host to register one.
-    INFLECTION_OVERRIDES = {}.freeze
+    # camelization, consulted by the inflections initializer below. An engine
+    # file that wants a genuine acronym in its constant adds its basename here
+    # rather than relying on the host to register one.
+    INFLECTION_OVERRIDES = {
+      "mcp_catalog" => "MCPCatalog",
+      "mcp_controller" => "MCPController",
+      "mcp_recording_middleware" => "MCPRecordingMiddleware",
+      "mcp_servers_controller" => "MCPServersController",
+      "playwright_mcp_client" => "PlaywrightMCPClient"
+    }.freeze
+
+    # The spelling default camelization produces for each overridden file,
+    # mapped to the constant the file actually defines. The const_missing shim
+    # below uses it to answer lookups that camelized with default inflections
+    # (a plain host's router, or caller code written against the pre-acronym
+    # names) with the acronym constant.
+    DEFAULT_SPELLINGS = INFLECTION_OVERRIDES.to_h { |basename, constant|
+      [ Zeitwerk::Inflector.new.camelize(basename, nil), constant ]
+    }.freeze
 
     config.action_agent = ActiveSupport::OrderedOptions.new
 
+    # Whether a request is a browser asking for a page, as opposed to an API
+    # or MCP client: the routes use it to tell the dashboard's client-side
+    # deep links apart from protocol traffic on the same path.
+    def self.html_request?(request)
+      Array(request.accepts).any? { |type| type.respond_to?(:html?) && type.html? }
+    rescue StandardError
+      false
+    end
+
     # The dashboard's JS and CSS ship prebuilt in the gem. Adding the
     # directory to the host app's asset paths is what lets a plain
     # `mount ActionAgent::Engine` work without the host running a
@@ -34,14 +57,13 @@ module ActionAgent
     end
 
     # This engine's constants are spelled the way Zeitwerk's own inflector
-    # spells them — Api, McpCatalog, ApiKey — but an engine's files are
+    # spells them — Api, ApiKey — but an engine's files are
     # autoloaded by the host's `rails.main` loader, under the *host's*
     # inflections. A host that declares `inflect.acronym "API"` or "MCP" (both
     # common, and documented by Rails) makes Zeitwerk expect
-    # ActionAgent::API::TracesController or ActionAgent::MCPCatalog from files
-    # that define ActionAgent::Api::TracesController and
-    # ActionAgent::McpCatalog. The constant never resolves and the request
-    # raises Zeitwerk::NameError.
+    # ActionAgent::API::TracesController from a file
+    # that defines ActionAgent::Api::TracesController. The constant never
+    # resolves and the request raises Zeitwerk::NameError.
     #
     # Every path under this engine therefore camelizes with Zeitwerk's default
     # rules, ignoring whatever acronyms the host has registered. Applied by
@@ -51,7 +73,8 @@ module ActionAgent
     # engine's files from the host's.
     #
     # Basenames whose spelling this engine cannot express through default
-    # camelization (a genuine acronym it wants uppercased) go in OVERRIDES.
+    # camelization (a genuine acronym it wants uppercased) go in
+    # INFLECTION_OVERRIDES.
     initializer "action_agent.inflections", before: :set_autoload_paths do
       engine_root = File.join(root.to_s, "")
       default = Zeitwerk::Inflector.new
@@ -74,15 +97,22 @@ module ActionAgent
     # So the namespace answers to both. `const_missing` rather than an eager
     # alias because the controllers are autoloaded on demand, and naming them at
     # boot would load the whole dashboard.
-    # The router does not consult the autoloader's inflector, so an acronym
-    # host asks for ActionAgent::API::MCPServersController while the constants
-    # are Api::McpServersController. Rather than enumerate the pairs, an
-    # all-caps run in a missing constant is retried in the spelling default
-    # camelization produces: API -> Api, MCPServersController -> McpServers-
-    # Controller. Only the engine's own namespaces are touched, and only for a
-    # constant that is already missing.
+    # The router does not consult the autoloader's inflector, so its lookups
+    # miss in both directions. An acronym host asks for
+    # ActionAgent::API::TracesController while the constant is
+    # Api::TracesController: an all-caps run in a missing constant is retried
+    # in the spelling default camelization produces (API -> Api). A plain host
+    # asks for Api::McpServersController while the constant is
+    # MCPServersController (the file is in INFLECTION_OVERRIDES): a missing
+    # constant matching an override's default spelling is retried as the
+    # acronym constant, via DEFAULT_SPELLINGS — which also keeps caller code
+    # written against the pre-acronym names resolving. Only the engine's own
+    # namespaces are touched, and only for a constant that is already missing.
     inflection_shim = Module.new do
       def const_missing(name)
+        acronym = ActionAgent::Engine::DEFAULT_SPELLINGS[name.to_s]
+        return const_get(acronym, false) if acronym && const_defined?(acronym, false)
+
         relaxed = name.to_s.gsub(/([A-Z])([A-Z]+)(?=[A-Z][a-z]|\d|\z)/) { "#{$1}#{$2.downcase}" }
 
         return super if relaxed == name.to_s || !const_defined?(relaxed, false)
@@ -102,11 +132,63 @@ module ActionAgent
       end
     end
 
+    # API keys and provider credentials are encrypted at rest (`encrypts` on
+    # ApiKey and ProviderKey), which needs Active Record Encryption keys the
+    # host app may never have generated — `rails db:encryption:init` is a
+    # step most installs skip, and the engine's own reference host had
+    # skipped it too. Without keys, every credential write and every
+    # authenticated MCP request raised Errors::Configuration as an HTML 500.
+    #
+    # Rails seeds ActiveRecord::Encryption from credentials and
+    # config.active_record.encryption in its own
+    # "active_record_encryption.configuration" initializer. Keys derived
+    # after that one land in config but never reach the encryptor, so the
+    # config reads back correct while every credential write raises
+    # Errors::Configuration — which is what a host that skipped
+    # db:encryption:init actually hits. Naming it in `before:` is what puts
+    # the derivation ahead of it. When the host set no keys — neither here
+    # nor in credentials — derive stable ones from secret_key_base, overridable
+    # through the ACTIVE_RECORD_ENCRYPTION_* variables, exactly as the
+    # activeagents.ai platform does. Host-provided keys are never
+    # overridden, and the derivation is stable as long as secret_key_base is.
+    initializer "action_agent.active_record_encryption", before: "active_record_encryption.configuration" do |app|
+      next unless ActionAgent.encrypt_credentials
+      next unless app.config.respond_to?(:active_record)
+
+      encryption = app.config.active_record.encryption
+      configured = %i[primary_key deterministic_key key_derivation_salt].any? { |key| encryption[key].present? }
+      configured ||= begin
+        app.credentials.dig(:active_record_encryption, :primary_key).present?
+      rescue StandardError
+        false
+      end
+      next if configured
+
+      derive = lambda do |purpose|
+        app.key_generator.generate_key("active_record_encryption.#{purpose}", 32).unpack1("H*")
+      end
+
+      encryption.primary_key = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"].presence || derive.call("primary_key")
+      encryption.deterministic_key = ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"].presence || derive.call("deterministic_key")
+      encryption.key_derivation_salt = ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"].presence || derive.call("key_derivation_salt")
+    end
+
     initializer "action_agent.assets", before: :append_assets_path do |app|
       builds = root.join("app", "assets", "builds").to_s
       next unless File.directory?(builds)
 
-      next unless app.config.respond_to?(:assets)
+      # The prebuilt bundles are served through the host's asset pipeline —
+      # propshaft (the Rails default) or sprockets-rails. A host without one
+      # (a `rails --api` app, or one that removed propshaft) used to get a
+      # silently blank dashboard with two 404s; say why instead.
+      unless app.config.respond_to?(:assets)
+        Rails.logger&.warn(
+          "[ActionAgent] the dashboard's assets cannot be served: this app has no asset pipeline. " \
+          "Add propshaft (or sprockets-rails) to the Gemfile so action_agent.js and action_agent.css " \
+          "are served from the engine's app/assets/builds."
+        )
+        next
+      end
 
       if app.config.assets.respond_to?(:paths)
         app.config.assets.paths << builds unless app.config.assets.paths.include?(builds)

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.2.2"
+  VERSION = "1.3.0"
 end

data/lib/action_agent.rb

@@ -83,7 +83,10 @@ require "action_agent/compatibility"
 #
 #   ActionAgent.configure do |config|
 #     config.authentication_method = ->(controller) { controller.authenticate_admin! }
-#     config.sandbox_service = :local  # Docker/Incus
+#     # Sandboxes run in the in-memory mock unless the app registers a real
+#     # backend (see sandbox_backends) and names it here:
+#     config.sandbox_backends = { "incus" => "IncusSandboxService" }
+#     config.sandbox_service = :incus
 #   end
 #
 # == Multi-tenant Mode
@@ -171,7 +174,10 @@ module ActionAgent
     # @return [String, nil]
     attr_accessor :layout
 
-    # Sandbox service type (:local, :cloud_run, :kubernetes)
+    # Which sandbox backend to provision with: :mock (the only one the
+    # engine ships — an in-memory fake that runs nothing) or the name of a
+    # backend the host registered in sandbox_backends. An unregistered name
+    # falls back to :mock with a logged warning.
     # @return [Symbol]
     attr_accessor :sandbox_service
 
@@ -217,9 +223,11 @@ module ActionAgent
     # @return [Proc, nil]
     attr_accessor :provider_credentials_resolver
 
-    # Extra sandbox backends contributed by the host app, as
-    # { "cloud_run" => "CloudRunService" }. The engine ships :mock and
-    # :local (Docker); cloud backends live in the app that operates them.
+    # Sandbox backends contributed by the host app, as
+    # { "cloud_run" => "CloudRunService" }. The engine ships only :mock;
+    # every real backend (Docker/Incus, Cloud Run, Kubernetes) lives in the
+    # app that operates it, which registers it here and selects it with
+    # sandbox_service.
     # @return [Hash{String => String}]
     attr_accessor :sandbox_backends
 
@@ -234,6 +242,37 @@ module ActionAgent
     # @return [String, nil]
     attr_accessor :upgrade_url
 
+    # The host app's sign-out endpoint, which the header's "Sign out" item
+    # POSTs to (with _method=delete and the CSRF token). The engine has no
+    # session of its own; unset, the menu item is not shown.
+    # @return [String, nil]
+    attr_accessor :sign_out_path
+
+    # Where a browser is sent when it asks for a dashboard page without a
+    # valid session — the host app's sign-in page. Unset, an unauthenticated
+    # page request gets a minimal session-expired page instead of a bare
+    # 401; API clients always get the 401.
+    # @return [String, nil]
+    attr_accessor :sign_in_path
+
+    # Answers GET <mount>/api/usage — the plan meter the Organization view
+    # and the Run Agents quota banner read. Receives (owner) and returns a
+    # Hash in the platform's shape:
+    #
+    #   { runs_used: 12, runs_limit: 100, runs_remaining: 88,
+    #     can_run: true, plan: "pro" }
+    #
+    # Unset means unlimited: the engine reports UNLIMITED_USAGE and the
+    # views hide the meter.
+    # @return [Proc, nil]
+    attr_accessor :usage_resolver
+
+    # What a dashboard with no usage_resolver reports: no limit, nothing
+    # counted, always allowed.
+    UNLIMITED_USAGE = {
+      runs_used: 0, runs_limit: nil, runs_remaining: nil, can_run: true, plan: nil, unlimited: true
+    }.freeze
+
     # Called after the dashboard performs a metered action, as
     # (owner, kind) — the counterpart to quota_checker, for host apps that
     # track usage against a plan. Unset means nothing is counted.
@@ -262,6 +301,16 @@ module ActionAgent
     # @return [Boolean]
     attr_accessor :encrypt_credentials
 
+    # MCP servers the host app itself serves or connects, appended to the
+    # built-in catalog (MCPCatalog) so the MCP Services view lists them and
+    # telemetry traffic attributes to them. Each entry is a hash shaped like
+    # a catalog entry: +key+ and +name+ at minimum, plus any of the optional
+    # fields (+description+, +transport+, +url+, +categories+, +docs_url+,
+    # +first_party+); +tool_hints+ names the bare tool names that belong to
+    # the server. A built-in entry keeps its key on collision.
+    # @return [Array<Hash>]
+    attr_accessor :mcp_catalog
+
     # Value stored in polymorphic *_type columns for dashboard agents
     # (agent_memories.memorable_type, agent_contexts.contextable_type).
     # Unset means the class name. A host app whose existing rows were
@@ -292,6 +341,19 @@ module ActionAgent
       nil
     end
 
+    # The usage meter for +owner+. Never raises: a bookkeeping failure must
+    # not take the views that display it down with it.
+    #
+    # @return [Hash] the platform's usage shape, UNLIMITED_USAGE by default
+    def usage_for(owner)
+      return UNLIMITED_USAGE.dup if usage_resolver.nil?
+
+      usage_resolver.call(owner) || UNLIMITED_USAGE.dup
+    rescue StandardError => e
+      Rails.logger.warn("[ActionAgent] usage lookup failed: #{e.message}")
+      UNLIMITED_USAGE.dup
+    end
+
     # Asks the host app whether +owner+ may perform +kind+.
     #
     # @return [String, Hash, nil] denial message or payload, nil when allowed
@@ -381,7 +443,7 @@ module ActionAgent
       @trace_model_class = nil
       @use_inertia = false
       @layout = nil
-      @sandbox_service = :local
+      @sandbox_service = :mock
       @sandbox_limits = nil
       @storage_service = nil
       @ingest_api_key = nil
@@ -396,7 +458,11 @@ module ActionAgent
       @trace_retention = nil
       @trace_owner_resolver = nil
       @usage_recorder = nil
+      @usage_resolver = nil
       @upgrade_url = nil
+      @sign_out_path = nil
+      @sign_in_path = nil
+      @mcp_catalog = []
     end
   end
 

data/lib/generators/action_agent/install_generator.rb

@@ -64,12 +64,20 @@ module ActionAgent
 
       if existing_migration?("create_active_agent_dashboard_tables")
         say_status :skip, "create_active_agent_dashboard_tables already exists", :yellow
-        return
+      else
+        migration_template(
+          "create_active_agent_dashboard_tables.rb.erb",
+          "db/migrate/create_active_agent_dashboard_tables.rb"
+        )
       end
 
+      # Scenario suites arrived after the dashboard tables shipped, so an
+      # install that already has those still needs this one.
+      return if existing_migration?("create_active_agent_evaluation_scenarios")
+
       migration_template(
-        "create_active_agent_dashboard_tables.rb.erb",
-        "db/migrate/create_active_agent_dashboard_tables.rb"
+        "create_active_agent_evaluation_scenarios.rb.erb",
+        "db/migrate/create_active_agent_evaluation_scenarios.rb"
       )
     end
 
@@ -102,10 +110,15 @@ module ActionAgent
       say "\n"
       say "Next steps:"
       say "  1. Run migrations: rails db:migrate"
-      say "  2. Configure telemetry in config/active_agent.yml:"
-      say "     telemetry:"
-      say "       enabled: true"
-      say "       local_storage: true"
+      # Nested under the environment key: ActiveAgent::Configuration.load
+      # reads only the current environment's subtree when the file has one,
+      # and the framework's own generator writes an env-keyed file — so a
+      # top-level telemetry: block is silently ignored.
+      say "  2. Configure telemetry in config/active_agent.yml, under each environment key:"
+      say "     development:"
+      say "       telemetry:"
+      say "         enabled: true"
+      say "         local_storage: true"
       say "  3. Visit /activeagents to view the dashboard"
       unless options[:traces_only]
         say "\n"

data/lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# Adds scenario suites to evaluations: the user-authored prompts an
+# evaluation replays through its agent (once per candidate model), and the
+# per-scenario, per-model result each replay produced. Emitted alongside the
+# dashboard tables on a fresh install, and on its own for an install that
+# predates scenario evaluations.
+#
+# Table names follow ActionAgent.table_name_prefix, and JSON columns are
+# jsonb on PostgreSQL and json elsewhere, the same way as
+# create_active_agent_dashboard_tables.
+class CreateActiveAgentEvaluationScenarios < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}evaluation_scenarios" do |t|
+      t.bigint :evaluation_id, null: false
+      t.string :key, null: false
+      t.string :group
+      t.text :prompt, null: false
+      t.text :notes
+      t.column :expectations, json_type, **json_default({})
+      t.integer :position, default: 0, null: false
+      t.boolean :enabled, default: true, null: false
+      t.timestamps
+      t.index [ :evaluation_id, :key ], unique: true, name: "index_#{prefix}evaluation_scenarios_on_evaluation_and_key"
+      t.index [ :evaluation_id, :group ], name: "index_#{prefix}evaluation_scenarios_on_evaluation_and_group"
+    end
+
+    create_table "#{prefix}evaluation_scenario_results" do |t|
+      t.bigint :evaluation_run_id, null: false
+      t.bigint :evaluation_scenario_id, null: false
+      t.bigint :agent_run_id
+      t.string :model, null: false
+      t.string :provider
+      t.integer :status, default: 0, null: false
+      t.float :score
+      t.column :scores, json_type, **json_default({})
+      t.text :output
+      t.column :tool_calls, json_type, **json_default([])
+      t.integer :duration_ms
+      t.integer :input_tokens
+      t.integer :output_tokens
+      t.decimal :cost, precision: 12, scale: 6
+      t.string :fault
+      t.text :recommendation
+      t.column :diagnosis, json_type, **json_default({})
+      t.text :error_message
+      t.timestamps
+      t.index :evaluation_run_id, name: "index_#{prefix}evaluation_scenario_results_on_run"
+      t.index :evaluation_scenario_id, name: "index_#{prefix}evaluation_scenario_results_on_scenario"
+      t.index [ :evaluation_run_id, :model ], name: "index_#{prefix}evaluation_scenario_results_on_run_and_model"
+    end
+
+    # Which scenarios and models a run covered, so a partial run (one group,
+    # one question) reads as such rather than as the whole suite.
+    unless column_exists?("#{prefix}evaluation_runs", :selection)
+      add_column "#{prefix}evaluation_runs", :selection, json_type, **json_default({})
+    end
+  end
+
+  private
+
+  def json_type
+    @json_type ||= postgres? ? :jsonb : :json
+  end
+
+  # MySQL rejects a default on a JSON column outright, so the column is
+  # created without one there.
+  def json_default(value, null: nil)
+    return {} unless postgres?
+
+    null.nil? ? { default: value } : { default: value, null: null }
+  end
+
+  def postgres?
+    connection.adapter_name.to_s.downcase.include?("postgres")
+  end
+end

data/lib/tasks/action_agent.rake

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+namespace :action_agent do
+  desc "Seed the dashboard's default agent templates (idempotent — existing slugs are kept)"
+  task seed_templates: :environment do
+    ActionAgent::AgentTemplate.seed_defaults!
+    puts "Agent template library: #{ActionAgent::AgentTemplate.count} templates"
+  end
+end

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Justin Bowen
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -15,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -25,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -142,11 +143,14 @@ files:
 - app/controllers/action_agent/api/tools_controller.rb
 - app/controllers/action_agent/api/trace_reports_controller.rb
 - app/controllers/action_agent/api/traces_controller.rb
+- app/controllers/action_agent/api/usage_controller.rb
 - app/controllers/action_agent/application_controller.rb
 - app/controllers/action_agent/dashboard_controller.rb
 - app/controllers/action_agent/traces_controller.rb
+- app/controllers/concerns/action_agent/api/agent_serialization.rb
 - app/jobs/action_agent/agent_execution_job.rb
 - app/jobs/action_agent/application_job.rb
+- app/jobs/action_agent/evaluation_run_job.rb
 - app/jobs/action_agent/process_telemetry_traces_job.rb
 - app/jobs/action_agent/sandbox_cleanup_job.rb
 - app/jobs/action_agent/sandbox_provision_job.rb
@@ -165,6 +169,8 @@ files:
 - app/models/action_agent/application_record.rb
 - app/models/action_agent/evaluation.rb
 - app/models/action_agent/evaluation_run.rb
+- app/models/action_agent/evaluation_scenario.rb
+- app/models/action_agent/evaluation_scenario_result.rb
 - app/models/action_agent/model_pricing.rb
 - app/models/action_agent/provider_key.rb
 - app/models/action_agent/recording_action.rb
@@ -178,6 +184,7 @@ files:
 - app/models/concerns/action_agent/ownable.rb
 - app/models/concerns/action_agent/session_recordable.rb
 - app/queries/action_agent/agent_executions.rb
+- app/queries/action_agent/metrics_report.rb
 - app/serializers/action_agent/agent_message_serializer.rb
 - app/serializers/action_agent/interaction_preview.rb
 - app/serializers/action_agent/telemetry_trace_serializer.rb
@@ -187,11 +194,13 @@ files:
 - app/services/action_agent/agent_scorecard.rb
 - app/services/action_agent/agent_toolbox.rb
 - app/services/action_agent/evaluation_runner_service.rb
+- app/services/action_agent/evaluation_tool_resolver.rb
 - app/services/action_agent/mcp_catalog.rb
 - app/services/action_agent/mcp_recording_middleware.rb
 - app/services/action_agent/mock_sandbox_backend.rb
 - app/services/action_agent/playwright_mcp_client.rb
 - app/services/action_agent/sandbox_orchestrator.rb
+- app/services/action_agent/scenario_evaluation_runner.rb
 - app/services/action_agent/session_recording_service.rb
 - app/services/action_agent/tool_discovery.rb
 - app/views/action_agent/dashboard/index.html.erb
@@ -211,7 +220,9 @@ files:
 - lib/generators/action_agent/templates/action_agent.rb.erb
 - lib/generators/action_agent/templates/add_agent_id_to_active_agent_telemetry_traces.rb.erb
 - lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb
+- lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb
 - lib/generators/action_agent/templates/create_active_agent_telemetry_traces.rb.erb
+- lib/tasks/action_agent.rake
 homepage: https://activeagents.ai
 licenses:
 - MIT
@@ -220,6 +231,7 @@ metadata:
   documentation_uri: https://docs.activeagents.ai/framework/self-hosted-observability
   source_code_uri: https://github.com/activeagents/activeagent
   rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -227,14 +239,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: The Active Agent dashboard, as a mountable Rails engine
 test_files: []