actionagent 1.6.4 → 1.7.1

data/app/services/action_agent/evaluation_runner_service.rb

@@ -15,8 +15,17 @@ module ActionAgent
       new(evaluation).call
     end
 
-    def initialize(evaluation)
+    # Returns the provider +owner+'s evaluation judge runs on (see
+    # #judge_provider), or nil when no provider has credentials.
+    def self.judge_provider_for(owner)
+      new(nil, owner: owner).judge_provider
+    end
+
+    # +owner+ is whose provider credentials the judge uses: the evaluated
+    # agent's owner unless given.
+    def initialize(evaluation, owner: nil)
       @evaluation = evaluation
+      @owner = owner
     end
 
     def call
@@ -55,6 +64,9 @@ module ActionAgent
         scores[criterion["key"]] = stats
       end
 
+      scores["_cohorts"] = cohort_summaries(samples, per_sample_scores) if samples.any?
+      scores["_judge_usage"] = judge_usage if judge_usage
+
       run.update!(
         status: :complete,
         scores: scores,
@@ -68,6 +80,17 @@ module ActionAgent
       raise
     end
 
+    # Returns the provider the judge runs on: the first of Anthropic, OpenAI
+    # and OpenRouter the owner or the host's config has a key for, else Ollama
+    # when the owner configured a host; nil when none. The judge runs
+    # `judge_model` as that provider's own model id.
+    def judge_provider
+      @judge_provider ||=
+        %i[anthropic openai openrouter].find do |name|
+          owner_provider_options(name).any? || global_provider_token?(name)
+        end || (:ollama if owner_provider_options(:ollama).any?)
+    end
+
     private
 
     # Scores each sample-based criterion once per candidate model cohort and
@@ -109,6 +132,11 @@ module ActionAgent
         scores["_verdict"] = verdict if verdict
       end
 
+      scores["_cohorts"] = active.to_h do |model, samples|
+        [ model, cohort_summary(samples, per_model_sample_scores[model] || {}) ]
+      end
+      scores["_judge_usage"] = judge_usage if judge_usage
+
       run.update!(
         status: :complete,
         scores: scores,
@@ -148,6 +176,98 @@ module ActionAgent
       end
     end
 
+    # --- Cohort summaries ------------------------------------------------------
+    #
+    # Stored under scores["_cohorts"], keyed by model: how many generations
+    # were sampled under it, how many cleared every criterion, their latency
+    # and token usage, and what those generations cost to serve. A comparison
+    # run gets one entry per requested cohort that had generations; a plain
+    # run one per model that happened to be among the sampled generations.
+    # This is what lets the dashboard show a run as "5/12 passed" per model,
+    # the way a scenario run's `_models` summaries do, without re-reading the
+    # generations.
+    #
+    # The cost here is the agent's: what the sampled interactions cost to
+    # operate. It is not what this run spent — scoring recorded generations
+    # costs nothing until a judge is asked, and the judge's own spend is
+    # recorded apart, under "_judge_usage" (see #judge_usage).
+
+    def cohort_summaries(samples, per_sample_scores)
+      samples
+        .group_by { |generation| generation.model.presence || "unknown" }
+        .transform_values { |group| cohort_summary(group, per_sample_scores) }
+    end
+
+    def cohort_summary(samples, per_sample_scores)
+      durations_ms = samples.filter_map do |generation|
+        seconds = generation.duration_seconds.to_f
+        seconds * 1000 if seconds.positive?
+      end
+      providers = samples.filter_map { |generation| generation.provider.presence }
+      input_tokens = samples.sum { |generation| generation.input_tokens.to_i }
+      output_tokens = samples.sum { |generation| generation.output_tokens.to_i }
+      costs = samples.filter_map do |generation|
+        ModelPricing.estimate(model: generation.model, input_tokens: generation.input_tokens, output_tokens: generation.output_tokens)
+      end
+
+      {
+        "samples" => samples.size,
+        "passed" => passed_count(per_sample_scores.slice(*samples.map(&:id))),
+        "provider" => providers.tally.max_by { |_provider, count| count }&.first,
+        "avg_duration_ms" => durations_ms.any? ? (durations_ms.sum / durations_ms.size).round : nil,
+        "input_tokens" => input_tokens,
+        "output_tokens" => output_tokens,
+        "cost" => costs.any? ? costs.sum.round(6) : nil
+      }
+    end
+
+    # --- Judge usage -----------------------------------------------------------
+    #
+    # Every call the judge makes is metered here, apart from the agent's own
+    # spend: scoring answers, recommending fixes, writing the verdict and,
+    # for a judge_defined evaluation, authoring the KPIs. The agent's cost is
+    # the operating figure — what serving these interactions costs — while
+    # the judge's is the evaluation's own, offline, agent-to-agent overhead,
+    # and a run that reported the two as one number would overstate the
+    # first. Persisted as scores["_judge_usage"]:
+    #
+    #   { "calls", "input_tokens", "output_tokens", "cost", "model",
+    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n } }
+    #
+    # nil until the judge has been asked something, so a rules-only run
+    # records no judge at all rather than a judge that cost nothing.
+
+    def judge_usage
+      return nil if @judge_usage.nil?
+
+      @judge_usage.merge("cost" => @judge_usage["cost"]&.round(6))
+    end
+
+    # Asks the judge and meters the answer. `kind` is the call's purpose —
+    # :score, :recommend, :verdict or :define — the same vocabulary
+    # ActiveAgent::Evals::Judge hands a block that accepts `kind:`.
+    def judge_generate(kind, message:, instructions:)
+      response = judge_class.prompt(message: message, instructions: instructions).generate_now
+      record_judge_call(kind, response)
+      response
+    end
+
+    def record_judge_call(kind, response)
+      usage = response.respond_to?(:usage) ? response.usage : nil
+      input_tokens = usage.respond_to?(:input_tokens) ? usage.input_tokens.to_i : 0
+      output_tokens = usage.respond_to?(:output_tokens) ? usage.output_tokens.to_i : 0
+      model = (response.respond_to?(:model) && response.model.presence) || @evaluation.judge_model.presence
+      cost = ModelPricing.estimate(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
+
+      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {} }
+      @judge_usage["calls"] += 1
+      @judge_usage["input_tokens"] += input_tokens
+      @judge_usage["output_tokens"] += output_tokens
+      @judge_usage["cost"] = (@judge_usage["cost"] || 0.0) + cost if cost
+      @judge_usage["model"] ||= model
+      @judge_usage["by_kind"][kind.to_s] = @judge_usage["by_kind"].fetch(kind.to_s, 0) + 1
+    end
+
     def sample_generations(model: nil)
       scope = @evaluation.agent.generations
       scope = scope.where(model: model) if model
@@ -175,7 +295,7 @@ module ActionAgent
       if total.zero?
         return {
           "skipped" => true,
-          "reason" => "No telemetry traces for #{@evaluation.agent.telemetry_agent_class} in the last #{window_hours}h"
+          "reason" => "No telemetry traces for #{telemetry_source} in the last #{window_hours}h"
         }
       end
 
@@ -213,12 +333,18 @@ module ActionAgent
     end
 
     def telemetry_traces(window_hours)
-      ActionAgent.trace_model
-        .for_account(ActionAgent.tenant_for(owner))
-        .for_agent(@evaluation.agent.telemetry_agent_class)
+      @evaluation.agent
+        .telemetry_traces(ActionAgent.trace_model.for_account(ActionAgent.tenant_for(owner)))
         .for_date_range(window_hours.hours.ago, Time.current)
     end
 
+    # What a skip reason says was looked for: the observed agent itself, or
+    # the class any other agent's traces are reported under.
+    def telemetry_source
+      agent = @evaluation.agent
+      agent.observed? ? agent.name : agent.telemetry_agent_class
+    end
+
     # Returns 0.0..1.0, or nil when the criterion cannot be scored.
     def score_sample(criterion, generation)
       config = criterion["config"] || {}
@@ -273,10 +399,11 @@ module ActionAgent
         raise "Judge-defined KPIs need provider credentials (add a provider API key in Settings)"
       end
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :define,
         message: kpi_definition_prompt,
         instructions: "You define measurable evaluation KPIs for AI agents. Respond ONLY with JSON."
-      ).generate_now
+      )
 
       kpis = parse_kpis(response.message&.content)
       raise "Judge returned no usable KPIs — try again or add criteria manually" if kpis.empty?
@@ -354,7 +481,8 @@ module ActionAgent
         "#{key}: #{cells.join(', ')}"
       end
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :verdict,
         message: <<~PROMPT,
           An AI agent was evaluated under multiple models. Its goals:
           ---
@@ -368,7 +496,7 @@ module ActionAgent
           Respond ONLY with JSON: {"winner": "<model>", "rationale": "<at most two sentences>"}
         PROMPT
         instructions: "You are an impartial evaluation judge comparing model cohorts. Respond ONLY with JSON."
-      ).generate_now
+      )
 
       json = response.message&.content.to_s[/\{.*\}/m]
       verdict = json ? JSON.parse(json) : nil
@@ -390,10 +518,11 @@ module ActionAgent
       return nil unless judge_available?
       return nil if generation.content.blank?
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :score,
         message: judge_prompt(criterion, generation),
         instructions: "You are an impartial evaluation judge. Respond ONLY with JSON: {\"score\": <float between 0.0 and 1.0>}"
-      ).generate_now
+      )
 
       parse_judge_score(response.message&.content)
     rescue StandardError => e
@@ -438,13 +567,6 @@ module ActionAgent
       judge_provider.present?
     end
 
-    def judge_provider
-      @judge_provider ||=
-        %i[anthropic openai openrouter].find do |name|
-          owner_provider_options(name).any? || global_provider_token?(name)
-        end || (:ollama if owner_provider_options(:ollama).any?)
-    end
-
     def global_provider_token?(name)
       config = ActiveAgent.configuration[name]
       config.respond_to?(:[]) && config[:access_token].present?
@@ -462,7 +584,7 @@ module ActionAgent
     end
 
     def owner
-      @owner ||= @evaluation.agent.owner
+      @owner ||= @evaluation&.agent&.owner
     end
 
     def judge_class

data/app/services/action_agent/evaluation_tool_resolver.rb

@@ -3,7 +3,7 @@
 module ActionAgent
   # Names the MCP server behind a tool an evaluation run needs, for the
   # report's fix items (ActiveAgent::Evals::Report#fix_items): a scenario
-  # that expected +search_slots+ and never got it is fixed by enabling the
+  # that expected +track_shipment+ and never got it is fixed by enabling the
   # server that serves it, and the item can only say so — and deep-link to
   # MCP Services — when something here can name that server.
   #
@@ -97,7 +97,7 @@ module ActionAgent
     end
 
     # normalized key => the display name a configured hash entry carries
-    # alongside its key ({"key" => "booking", "name" => "Booking Service"}).
+    # alongside its key ({"key" => "shipping", "name" => "Shipping Desk"}).
     def configured_names
       @configured_names ||= configured_entries.each_with_object({}) do |entry, map|
         next unless entry.respond_to?(:key?)
@@ -111,7 +111,7 @@ module ActionAgent
     end
 
     # bare tool name => server key, from configured entries that list the
-    # tools they serve ({"name" => "booking", "tools" => ["search_slots"]}),
+    # tools they serve ({"name" => "shipping", "tools" => ["track_shipment"]}),
     # in the catalog's own +tool_hints+ spelling or as tool hashes.
     def configured_tools
       @configured_tools ||= configured_entries.each_with_object({}) do |entry, map|

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -21,6 +21,11 @@ module ActionAgent
   class ScenarioEvaluationRunner < EvaluationRunnerService
     Evals = ActiveAgent::Evals
 
+    # The providers a candidate model name resolves against. `mock` is the
+    # framework's test double, accepted so the test suite can compare cohorts
+    # offline.
+    CANDIDATE_PROVIDERS = (Agent::PROVIDERS + %w[mock]).freeze
+
     # `run` is an EvaluationRun created ahead of time (by run_later!, so the
     # UI can show it pending while the job waits); absent, one is created here.
     def self.call(evaluation, selection: {}, run: nil)
@@ -126,15 +131,14 @@ module ActionAgent
     end
 
     # The models to compare: an explicit selection, else the evaluation's
-    # compare_models, else the agent as configured. `mock` is the framework's
-    # test double, accepted so the test suite can compare cohorts offline.
+    # compare_models, else the agent as configured.
     def model_specs
       names = Array(@selection[:models]).presence || @evaluation.compare_models
-      specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
+      specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: CANDIDATE_PROVIDERS)
       # parse_all resolves a bare name against `providers:` but passes through a
       # `provider/model` whose provider is not in that list, so the run would
       # otherwise reach the replay with a provider nothing can serve.
-      unsupported = specs.map(&:provider).uniq - (Agent::PROVIDERS + %w[mock])
+      unsupported = specs.map(&:provider).uniq - CANDIDATE_PROVIDERS
       raise ArgumentError, "unsupported model provider: #{unsupported.to_sentence}" if unsupported.any?
 
       return specs if specs.any?
@@ -261,6 +265,10 @@ module ActionAgent
       scores["_selection"] = run.selection
       scores["_metadata"] = report.metadata
       scores["_judge_label"] = report.judge_label || report.judge&.label
+      # The judge's own spend, apart from the replays' (which the results
+      # carry and EvaluationRun#usage sums). A host adapter that runs its own
+      # judge is out of reach of this meter, so its runs record none.
+      scores["_judge_usage"] = judge_usage if judge_usage
       scores
     end
 
@@ -318,12 +326,13 @@ module ActionAgent
 
     # The judge the evaluation's owner has credentials for, wrapped for the
     # evaluation core; nil when none is configured, in which case scoring
-    # stays on rules and expectations.
+    # stays on rules and expectations. The block takes `kind:` so each call
+    # is metered under what it was for (EvaluationRunnerService#judge_generate).
     def evals_judge
       return nil unless judge_available?
 
-      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:|
-        judge_class.prompt(message: prompt, instructions: instructions).generate_now.message&.content
+      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:, kind:|
+        judge_generate(kind, message: prompt, instructions: instructions).message&.content
       end
     end
   end

data/config/routes.rb

@@ -27,6 +27,11 @@ ActionAgent::Engine.routes.draw do
     # Authenticated with a bearer token, not a session.
     resources :traces, only: [ :create ]
 
+    # The collector for evaluation reports an application ran itself
+    # (ActiveAgent::Evals::Publisher), at <mount>/api/evaluation_reports.
+    # Authenticated like trace ingest, with a bearer token.
+    resources :evaluation_reports, only: [ :create ]
+
     # A JSON API has no :new or :edit forms to serve.
     resources :agents, except: [ :new, :edit ] do
       member do
@@ -45,6 +50,9 @@ ActionAgent::Engine.routes.draw do
         # and a fresh one to pin a first message to.
         get :conversations
         post :conversations, action: :create_conversation
+        # The evaluation form's models field: the model names this agent's
+        # generations were recorded under.
+        get :recorded_models
       end
       collection do
         get :presets

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.6.4"
+  VERSION = "1.7.1"
 end

data/lib/action_agent.rb

@@ -209,24 +209,77 @@ module ActionAgent
     # @return [Object, nil] Object responding to #signed_url_for and #fetch_snapshot
     attr_accessor :storage_service
 
-    # Bearer token required by the ingest API in single-tenant mode. When
-    # unset the local ingest endpoint accepts unauthenticated posts, so set
-    # it whenever the mount is reachable beyond your own machine.
-    # (Multi-tenant mode authenticates per-account keys instead.)
+    # Bearer token required in single-tenant mode by the endpoints other
+    # applications post to: trace ingest (<mount>/api/traces) and published
+    # evaluation reports (<mount>/api/evaluation_reports). When unset both
+    # accept unauthenticated posts, so set it whenever the mount is reachable
+    # beyond your own machine. Trace ingest also takes a form post, which any
+    # web page open in a browser on that machine can send it; the report
+    # collector takes only application/json, which a page cannot send
+    # cross-site. (Multi-tenant mode authenticates per-account keys instead.)
     # @return [String, nil]
     attr_accessor :ingest_api_key
 
+    # Concerns included into ActionAgent::ApplicationRecord as it loads, and
+    # through it into every engine model. An entry is a Module or the name
+    # of one. A name is resolved when the class
+    # loads, so an initializer can refer to a constant the host has not
+    # autoloaded yet, and a name that resolves to nothing raises NameError
+    # there rather than being skipped.
+    #
+    #   ActionAgent.configure do |config|
+    #     config.model_concerns = ["MyApp::ConnectionSwitching"]
+    #   end
+    #
+    # The class loads after the initializers have run, so set this in an
+    # initializer; a concern added later is not applied.
+    # @return [Array<Module, String>]
+    attr_accessor :model_concerns
+
+    # Concerns included into ActionAgent::ApplicationController as it loads,
+    # and through it into every dashboard controller: the React dashboard
+    # and its JSON API, the server-rendered console and the MCP facade.
+    # They are included ahead of the engine's own callbacks, so a concern's
+    # before_action or around_action runs before the dashboard
+    # authenticates. Entries are Modules or names, as for model_concerns.
+    #
+    # Not the endpoints other applications post to: Api::TracesController
+    # and Api::EvaluationReportsController authenticate with a bearer token
+    # and inherit ActionController::API.
+    #
+    #   ActionAgent.configure do |config|
+    #     config.controller_concerns = ["MyApp::RequestTagging"]
+    #   end
+    # @return [Array<Module, String>]
+    attr_accessor :controller_concerns
+
     # @deprecated Never consumed — dashboard controllers inherit
-    #   ActionController::Base. Retained as a no-op so existing
-    #   initializers that set it keep booting; remove in the next major.
+    #   ActionController::Base, and controller_concerns is how a host puts
+    #   its own behaviour on them. Assigning it warns and stores a value
+    #   nothing reads; removed in the next major.
     # @return [String]
-    attr_accessor :base_controller_class
+    attr_reader :base_controller_class
+
+    def base_controller_class=(value)
+      deprecator.warn(
+        "ActionAgent.base_controller_class has never been consumed and is removed in 2.0. " \
+        "Set ActionAgent.controller_concerns to extend the dashboard's controllers."
+      )
+      @base_controller_class = value
+    end
 
-    # Called before each run/trace-ingest to enforce host-app limits.
-    # Receives (owner, kind) where kind is :execution or :trace_ingest, and
-    # returns nil to allow, or to deny: a message String, or a Hash merged
-    # into the response so the app can surface its own usage numbers.
-    # Denials surface as HTTP 402 (execution) / 429 (ingest).
+    # Called before each metered action to enforce host-app limits.
+    # Receives (owner, kind) and returns nil to allow, or to deny: a message
+    # String, or a Hash merged into the response so the app can surface its
+    # own usage numbers. The kinds, and how a denial surfaces:
+    #
+    #   :execution         — an agent run; HTTP 402
+    #   :trace_ingest      — a POST to <mount>/api/traces; HTTP 429
+    #   :evaluation_report — a report <mount>/api/evaluation_reports would
+    #                        store (never an identical retry); HTTP 429
+    #
+    # The owner of an ingest kind is the tenant the key resolved to, nil on a
+    # single-tenant install.
     #
     # Unset means unlimited, which is what a self-hosted install wants.
     # @return [Proc, nil]
@@ -256,6 +309,21 @@ module ActionAgent
     # @return [Boolean]
     attr_accessor :execution_enabled
 
+    # Whether a run of an agent that mirrors a host class executes that class,
+    # instead of the class the engine builds from the record's `tools` and
+    # `instructions` columns.
+    #
+    # Off by default: it changes what a run of a mirrored agent executes, and
+    # a host that has tuned its dashboard records around the dynamic runtime
+    # should opt in deliberately. Dashboard-authored agents — the ones with no
+    # `agent_class_name` — are unaffected either way.
+    #
+    # On, a mirrored agent runs its real tools, delegations and instructions,
+    # so an evaluation scores the agent production runs rather than a
+    # flattened copy of it.
+    # @return [Boolean]
+    attr_accessor :run_host_agent_classes
+
     # Whether the "Ask ActiveAgents" assistant is available.
     #
     # The assistant is a tool for developing and CI-ing agents: it sends
@@ -314,7 +382,9 @@ module ActionAgent
 
     # Called after the dashboard performs a metered action, as
     # (owner, kind) — the counterpart to quota_checker, for host apps that
-    # track usage against a plan. Unset means nothing is counted.
+    # track usage against a plan. The kinds are :execution, for each agent
+    # run, and :evaluation_report, for each report the collector stores; an
+    # identical retry is not counted again. Unset means nothing is counted.
     # @return [Proc, nil]
     attr_accessor :usage_recorder
 
@@ -323,6 +393,11 @@ module ActionAgent
     # nobody in single-tenant mode. A host app whose agents hang off a
     # different record (the platform's hang off the account's owning user)
     # supplies its own mapping.
+    #
+    # A published evaluation report's agent is placed the same way: the
+    # resolver receives an unsaved trace with the publishing tenant as its
+    # account, and the report's source and agent name as its service_name and
+    # agent_class. In multi-tenant mode it must not return nil there.
     # @return [Proc, nil]
     attr_accessor :trace_owner_resolver
 
@@ -513,6 +588,19 @@ module ActionAgent
       name&.safe_constantize
     end
 
+    # The modules model_concerns names. ApplicationRecord reads it as it loads.
+    # @return [Array<Module>]
+    def model_concern_modules
+      resolve_concerns(model_concerns)
+    end
+
+    # The modules controller_concerns names. ApplicationController reads it
+    # as it loads.
+    # @return [Array<Module>]
+    def controller_concern_modules
+      resolve_concerns(controller_concerns)
+    end
+
     # Configures the dashboard.
     #
     # @yield [config] Configuration block
@@ -540,10 +628,13 @@ module ActionAgent
       @storage_service = nil
       @ingest_api_key = nil
       @base_controller_class = "ActionController::Base" # deprecated no-op
+      @model_concerns = []
+      @controller_concerns = []
       @quota_checker = nil
       @provider_credentials_resolver = nil
       @sandbox_backends = {}
       @execution_enabled = true
+      @run_host_agent_classes = false
       @assistant_enabled = nil
 
       @scenario_evaluation_adapter_resolver = nil
@@ -639,6 +730,12 @@ module ActionAgent
     def schema_tool_class_for(name)
       schema_tool_classes.find { |klass| klass.tool?(name) }
     end
+
+    private
+
+    def resolve_concerns(entries)
+      Array(entries).map { |entry| entry.is_a?(Module) ? entry : entry.to_s.constantize }
+    end
   end
 
   # Set defaults

data/lib/generators/action_agent/install_generator.rb

@@ -77,13 +77,36 @@ module ActionAgent
         )
       end
 
+      # add_agent_releases is emitted before the dashboard tables, so on an
+      # install generated fresh before the create-table migration carried the
+      # release columns it skipped every table, and its copy may name tables
+      # without the configured prefix. This adds whatever is still missing,
+      # and changes nothing where the columns exist.
+      unless existing_migration?("ensure_agent_release_columns")
+        migration_template(
+          "ensure_agent_release_columns.rb.erb",
+          "db/migrate/ensure_agent_release_columns.rb"
+        )
+      end
+
       # Scenario suites arrived after the dashboard tables shipped, so an
       # install that already has those still needs this one.
-      return if existing_migration?("create_active_agent_evaluation_scenarios")
+      unless existing_migration?("create_active_agent_evaluation_scenarios")
+        migration_template(
+          "create_active_agent_evaluation_scenarios.rb.erb",
+          "db/migrate/create_active_agent_evaluation_scenarios.rb"
+        )
+      end
+
+      # Published evaluation reports arrived after the dashboard tables
+      # shipped. Emitted after them, so on a fresh install it runs once the
+      # table exists, finds the columns the create-table migration made, and
+      # changes nothing.
+      return if existing_migration?("add_evaluation_report_identity")
 
       migration_template(
-        "create_active_agent_evaluation_scenarios.rb.erb",
-        "db/migrate/create_active_agent_evaluation_scenarios.rb"
+        "add_evaluation_report_identity.rb.erb",
+        "db/migrate/add_evaluation_report_identity.rb"
       )
     end
 

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -44,9 +44,11 @@ ActionAgent.configure do |config|
   # Ingest API authentication
   # ==========================================================================
   #
-  # Bearer token other apps must send when posting traces to the mounted
-  # ingest endpoint (/activeagents/api/traces). Leave unset only when the
-  # mount is not reachable beyond your own machine.
+  # Bearer token other apps must send when posting traces
+  # (/activeagents/api/traces) or evaluation reports
+  # (/activeagents/api/evaluation_reports) to the mount. Leave unset only
+  # when the mount is not reachable beyond your own machine; even then, any web
+  # page open in a browser there can post form data to the trace endpoint.
   #
   # config.ingest_api_key = Rails.application.credentials.dig(:active_agent, :ingest_api_key)
 
@@ -78,6 +80,20 @@ ActionAgent.configure do |config|
   # nothing rather than everything. An empty dashboard for a signed-in user
   # means the resolver above returned nil.
 
+  # ==========================================================================
+  # Host integration
+  # ==========================================================================
+  #
+  # Concerns your own models and controllers carry, applied to the engine's.
+  # model_concerns are included into ActionAgent::ApplicationRecord, and so
+  # into every engine model. controller_concerns are included into
+  # ActionAgent::ApplicationController ahead of its own callbacks, and so
+  # into every dashboard controller — not the bearer-token ingest endpoints.
+  # Modules or their names; a name is resolved when the class loads.
+  #
+  # config.model_concerns = ["ConnectionSwitching"]
+  # config.controller_concerns = ["RequestTagging"]
+
   # ==========================================================================
   # Ask ActiveAgents assistant
   # ==========================================================================