actionagent 1.7.0 → 1.7.1

data/app/controllers/action_agent/api/agents_controller.rb

@@ -18,6 +18,8 @@ module ActionAgent
       DEFAULT_LIST_SORT = "recent"
       # Conversations returned to the runner's picker when no limit is asked for.
       CONVERSATIONS_LIMIT = 50
+      # Model names returned by #recorded_models.
+      RECORDED_MODELS_LIMIT = 50
       # Keywords Agent#execute takes in its own right, which per-run overrides
       # must never supply (see #execution_params). `actor` is here for the
       # same reason as the rest and one more: a keyword splat wins over the
@@ -27,7 +29,7 @@ module ActionAgent
 
       before_action :set_agent, only: [
         :show, :update, :destroy, :versions, :runs, :execute, :test, :restore, :duplicate, :export, :analytics,
-        :tool_roster, :conversations, :create_conversation
+        :tool_roster, :conversations, :create_conversation, :recorded_models
       ]
       before_action :require_execution_enabled!, only: [ :execute, :test ]
       before_action :require_owner!, only: [ :execute, :test ]
@@ -274,6 +276,23 @@ module ActionAgent
         ).as_json
       end
 
+      # GET /api/agents/:id/recorded_models
+      #
+      # The model names this agent's generations were recorded under, most
+      # recently used first. An evaluation without scenarios compares the
+      # generations recorded under each name it is given, so these are the
+      # names its models field suggests.
+      def recorded_models
+        generations = AgentGeneration.arel_table
+        models = @agent.generations.where.not(model: [ nil, "" ])
+          .group(generations[:model])
+          .order(Arel::Nodes::Descending.new(generations[:created_at].maximum))
+          .limit(RECORDED_MODELS_LIMIT)
+          .pluck(generations[:model])
+
+        render json: { models: models }
+      end
+
       # GET /api/agents/:id/analytics
       #
       # Every execution of this agent, whoever ran it — the same merged model

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -38,12 +38,28 @@ module ActionAgent
       # 50 client-side hides an agent whose evaluations are not among the account's
       # 50 most recent. The scope is already restricted to the current user's
       # agents, so an id outside it simply returns nothing.
+      #
+      # Three fields feed the dashboard's model pickers. They describe the
+      # credentials of #picker_credentials_owner:
+      #   - judge_provider:        the provider a judge model runs on, null
+      #                            when none has credentials or the lookup
+      #                            raised
+      #   - judge_provider_error:  true when that lookup raised, as it does
+      #                            when a stored key no longer decrypts
+      #   - model_providers:       the providers agent runs have credentials
+      #                            for, leaving out any whose credentials
+      #                            cannot be read
       def index
         scope = evaluations_scope
         scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present?
         evaluations = scope.includes(:agent, :evaluation_runs, :scenarios).recent.limit(50)
+        owner = picker_credentials_owner
 
-        render json: { evaluations: evaluations.map { |evaluation| serialize(evaluation) } }
+        render json: {
+          evaluations: evaluations.map { |evaluation| serialize(evaluation) },
+          **judge_provider_fields(owner),
+          model_providers: AgentExecutionService.available_providers(owner)
+        }
       end
 
       # Runs listed per evaluation on GET /api/evaluations/:id. The rest of
@@ -224,6 +240,25 @@ module ActionAgent
           evaluation.evaluation_runs.create!(status: :failed, error_message: e.message, completed_at: Time.current)
       end
 
+      # Returns whose credentials the index's model picker fields describe.
+      # Agent runs and their judge use the evaluated agent's owner's
+      # credentials, so a list scoped to one agent reads that agent's owner,
+      # and an unscoped list the signed-in owner.
+      def picker_credentials_owner
+        agent = owner_agents.find_by(id: params[:agent_id]) if params[:agent_id].present?
+        agent ? agent.owner : current_owner
+      end
+
+      # Returns the index's judge_provider and judge_provider_error for
+      # +owner+. A lookup that raises, from a key that no longer decrypts or a
+      # host credentials hook that fails, is logged and reported as an error.
+      def judge_provider_fields(owner)
+        { judge_provider: EvaluationRunnerService.judge_provider_for(owner)&.to_s, judge_provider_error: false }
+      rescue StandardError => e
+        Rails.logger.warn("[Evaluations] judge provider lookup failed: #{e.class}: #{e.message}")
+        { judge_provider: nil, judge_provider_error: true }
+      end
+
       def evaluations_scope
         Evaluation.joins(:agent).where(agent: owner_agents)
       end

data/app/controllers/action_agent/api/provider_models_controller.rb

@@ -2,14 +2,22 @@
 
 module ActionAgent
   module Api
-    # Model catalogs for the agent builder/editor dropdowns.
+    # Model catalogs for the dashboard's model pickers: the agent builder and
+    # editor, and the evaluation forms.
     #
     # Hosted providers get a curated list of current models (kept here, server
     # side, so the UI can't drift stale). Ollama is queried live from the
     # account's configured host (Settings -> Provider API Keys, falling back to
     # the platform config) so locally pulled models appear; OpenRouter is
-    # queried from its public catalog. Live lookups fall back to the curated
-    # list on any failure.
+    # queried from its public catalog, and Anthropic from its Models API with
+    # the account's key. Live lookups fall back to the curated list on any
+    # failure.
+    #
+    # When the host app loads RubyLLM, the chat models its registry lists for
+    # the provider that take and return text follow: RubyLLM's bundled
+    # catalog, or the host's own model table when it configured one. The list
+    # stays de-duplicated, and its first id, the builder's preselected
+    # default, is always the live or curated one.
     class ProviderModelsController < BaseController
       before_action :require_owner!
 
@@ -42,7 +50,7 @@ module ActionAgent
           source = "curated"
         end
 
-        render json: { provider: provider, models: models, source: source }
+        render json: { provider: provider, models: (models + registry_models(provider)).uniq, source: source }
       end
 
       private
@@ -106,6 +114,29 @@ module ActionAgent
         nil
       end
 
+      # Returns the ids of the chat models RubyLLM's registry lists for
+      # +provider+ that take and return text. Empty without RubyLLM, or when
+      # the registry raises. This engine's provider names are RubyLLM's own
+      # slugs.
+      #
+      # `chat_models` alone is not enough: RubyLLM counts a model as a chat
+      # model whenever its output modalities name no other kind, none at all
+      # included. That takes in the speech, transcription, moderation and
+      # completion-only models its registry lists with no modalities, and the
+      # transcription models it lists as taking audio. The few chat models it
+      # lists with no modalities are left out with them.
+      def registry_models(provider)
+        return [] unless defined?(::RubyLLM) && ::RubyLLM.respond_to?(:models)
+
+        ::RubyLLM.models.by_provider(provider).chat_models.filter_map do |model|
+          modalities = model.modalities
+          model.id.presence if Array(modalities&.input).include?("text") && Array(modalities&.output).include?("text")
+        end
+      rescue StandardError => e
+        Rails.logger.warn("[ProviderModels] RubyLLM registry lookup failed: #{e.message}")
+        []
+      end
+
       def fetch_json(uri)
         response = Net::HTTP.start(
           uri.host, uri.port,

data/app/services/action_agent/agent_execution_service.rb

@@ -49,6 +49,12 @@ module ActionAgent
       new(agent_record, run).call
     end
 
+    # Returns the providers in Agent::PROVIDERS a run on +owner+'s behalf has
+    # credentials for, in that order (see #available_providers).
+    def self.available_providers(owner)
+      new(nil, nil, owner: owner).available_providers
+    end
+
     # Tool-call keywords that name the caller. The model's arguments and the
     # run's actor share one keyword namespace by the time they reach a tool,
     # so anything a model emits under these names is dropped before the call:
@@ -56,13 +62,23 @@ module ActionAgent
     # documents a model reads are attacker-reachable.
     ACTOR_KEYWORDS = %i[actor current_user].freeze
 
-    def initialize(agent_record, run)
+    # +owner+ is whose provider credentials the run uses: the agent record's
+    # owner unless given.
+    def initialize(agent_record, run, owner: nil)
       @agent_record = agent_record
       @run = run
+      @owner = owner
       @tool_invocations = []
       @event_sequence = 0
     end
 
+    # Returns the providers in Agent::PROVIDERS the owner's credentials, or
+    # the host's config, let a run use: #provider_available? for each.
+    # @return [Array<String>]
+    def available_providers
+      Agent::PROVIDERS.select { |name| provider_available?(name) }
+    end
+
     # The caller this run executes on behalf of, or nil when it runs
     # unattributed. Passed to every tool as +actor:+ — a host's SchemaTools
     # scope block, Pundit policy or agent callback decides what that means.
@@ -943,7 +959,7 @@ module ActionAgent
     # The agent's owner under the configured mode; nil when the install
     # has no owner model at all.
     def owner
-      @owner ||= @agent_record.owner
+      @owner ||= @agent_record&.owner
     end
 
     def record_trace(root_span)

data/app/services/action_agent/evaluation_runner_service.rb

@@ -15,8 +15,17 @@ module ActionAgent
       new(evaluation).call
     end
 
-    def initialize(evaluation)
+    # Returns the provider +owner+'s evaluation judge runs on (see
+    # #judge_provider), or nil when no provider has credentials.
+    def self.judge_provider_for(owner)
+      new(nil, owner: owner).judge_provider
+    end
+
+    # +owner+ is whose provider credentials the judge uses: the evaluated
+    # agent's owner unless given.
+    def initialize(evaluation, owner: nil)
       @evaluation = evaluation
+      @owner = owner
     end
 
     def call
@@ -71,6 +80,17 @@ module ActionAgent
       raise
     end
 
+    # Returns the provider the judge runs on: the first of Anthropic, OpenAI
+    # and OpenRouter the owner or the host's config has a key for, else Ollama
+    # when the owner configured a host; nil when none. The judge runs
+    # `judge_model` as that provider's own model id.
+    def judge_provider
+      @judge_provider ||=
+        %i[anthropic openai openrouter].find do |name|
+          owner_provider_options(name).any? || global_provider_token?(name)
+        end || (:ollama if owner_provider_options(:ollama).any?)
+    end
+
     private
 
     # Scores each sample-based criterion once per candidate model cohort and
@@ -547,13 +567,6 @@ module ActionAgent
       judge_provider.present?
     end
 
-    def judge_provider
-      @judge_provider ||=
-        %i[anthropic openai openrouter].find do |name|
-          owner_provider_options(name).any? || global_provider_token?(name)
-        end || (:ollama if owner_provider_options(:ollama).any?)
-    end
-
     def global_provider_token?(name)
       config = ActiveAgent.configuration[name]
       config.respond_to?(:[]) && config[:access_token].present?
@@ -571,7 +584,7 @@ module ActionAgent
     end
 
     def owner
-      @owner ||= @evaluation.agent.owner
+      @owner ||= @evaluation&.agent&.owner
     end
 
     def judge_class

data/app/services/action_agent/evaluation_tool_resolver.rb

@@ -3,7 +3,7 @@
 module ActionAgent
   # Names the MCP server behind a tool an evaluation run needs, for the
   # report's fix items (ActiveAgent::Evals::Report#fix_items): a scenario
-  # that expected +search_slots+ and never got it is fixed by enabling the
+  # that expected +track_shipment+ and never got it is fixed by enabling the
   # server that serves it, and the item can only say so — and deep-link to
   # MCP Services — when something here can name that server.
   #
@@ -97,7 +97,7 @@ module ActionAgent
     end
 
     # normalized key => the display name a configured hash entry carries
-    # alongside its key ({"key" => "booking", "name" => "Booking Service"}).
+    # alongside its key ({"key" => "shipping", "name" => "Shipping Desk"}).
     def configured_names
       @configured_names ||= configured_entries.each_with_object({}) do |entry, map|
         next unless entry.respond_to?(:key?)
@@ -111,7 +111,7 @@ module ActionAgent
     end
 
     # bare tool name => server key, from configured entries that list the
-    # tools they serve ({"name" => "booking", "tools" => ["search_slots"]}),
+    # tools they serve ({"name" => "shipping", "tools" => ["track_shipment"]}),
     # in the catalog's own +tool_hints+ spelling or as tool hashes.
     def configured_tools
       @configured_tools ||= configured_entries.each_with_object({}) do |entry, map|

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -21,6 +21,11 @@ module ActionAgent
   class ScenarioEvaluationRunner < EvaluationRunnerService
     Evals = ActiveAgent::Evals
 
+    # The providers a candidate model name resolves against. `mock` is the
+    # framework's test double, accepted so the test suite can compare cohorts
+    # offline.
+    CANDIDATE_PROVIDERS = (Agent::PROVIDERS + %w[mock]).freeze
+
     # `run` is an EvaluationRun created ahead of time (by run_later!, so the
     # UI can show it pending while the job waits); absent, one is created here.
     def self.call(evaluation, selection: {}, run: nil)
@@ -126,15 +131,14 @@ module ActionAgent
     end
 
     # The models to compare: an explicit selection, else the evaluation's
-    # compare_models, else the agent as configured. `mock` is the framework's
-    # test double, accepted so the test suite can compare cohorts offline.
+    # compare_models, else the agent as configured.
     def model_specs
       names = Array(@selection[:models]).presence || @evaluation.compare_models
-      specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
+      specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: CANDIDATE_PROVIDERS)
       # parse_all resolves a bare name against `providers:` but passes through a
       # `provider/model` whose provider is not in that list, so the run would
       # otherwise reach the replay with a provider nothing can serve.
-      unsupported = specs.map(&:provider).uniq - (Agent::PROVIDERS + %w[mock])
+      unsupported = specs.map(&:provider).uniq - CANDIDATE_PROVIDERS
       raise ArgumentError, "unsupported model provider: #{unsupported.to_sentence}" if unsupported.any?
 
       return specs if specs.any?

data/config/routes.rb

@@ -50,6 +50,9 @@ ActionAgent::Engine.routes.draw do
         # and a fresh one to pin a first message to.
         get :conversations
         post :conversations, action: :create_conversation
+        # The evaluation form's models field: the model names this agent's
+        # generations were recorded under.
+        get :recorded_models
       end
       collection do
         get :presets

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.7.0"
+  VERSION = "1.7.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-24 00:00:00.000000000 Z
+date: 2026-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent