actionagent 1.6.4 → 1.7.0

data/app/services/action_agent/evaluation_runner_service.rb

@@ -55,6 +55,9 @@ module ActionAgent
         scores[criterion["key"]] = stats
       end
 
+      scores["_cohorts"] = cohort_summaries(samples, per_sample_scores) if samples.any?
+      scores["_judge_usage"] = judge_usage if judge_usage
+
       run.update!(
         status: :complete,
         scores: scores,
@@ -109,6 +112,11 @@ module ActionAgent
         scores["_verdict"] = verdict if verdict
       end
 
+      scores["_cohorts"] = active.to_h do |model, samples|
+        [ model, cohort_summary(samples, per_model_sample_scores[model] || {}) ]
+      end
+      scores["_judge_usage"] = judge_usage if judge_usage
+
       run.update!(
         status: :complete,
         scores: scores,
@@ -148,6 +156,98 @@ module ActionAgent
       end
     end
 
+    # --- Cohort summaries ------------------------------------------------------
+    #
+    # Stored under scores["_cohorts"], keyed by model: how many generations
+    # were sampled under it, how many cleared every criterion, their latency
+    # and token usage, and what those generations cost to serve. A comparison
+    # run gets one entry per requested cohort that had generations; a plain
+    # run one per model that happened to be among the sampled generations.
+    # This is what lets the dashboard show a run as "5/12 passed" per model,
+    # the way a scenario run's `_models` summaries do, without re-reading the
+    # generations.
+    #
+    # The cost here is the agent's: what the sampled interactions cost to
+    # operate. It is not what this run spent — scoring recorded generations
+    # costs nothing until a judge is asked, and the judge's own spend is
+    # recorded apart, under "_judge_usage" (see #judge_usage).
+
+    def cohort_summaries(samples, per_sample_scores)
+      samples
+        .group_by { |generation| generation.model.presence || "unknown" }
+        .transform_values { |group| cohort_summary(group, per_sample_scores) }
+    end
+
+    def cohort_summary(samples, per_sample_scores)
+      durations_ms = samples.filter_map do |generation|
+        seconds = generation.duration_seconds.to_f
+        seconds * 1000 if seconds.positive?
+      end
+      providers = samples.filter_map { |generation| generation.provider.presence }
+      input_tokens = samples.sum { |generation| generation.input_tokens.to_i }
+      output_tokens = samples.sum { |generation| generation.output_tokens.to_i }
+      costs = samples.filter_map do |generation|
+        ModelPricing.estimate(model: generation.model, input_tokens: generation.input_tokens, output_tokens: generation.output_tokens)
+      end
+
+      {
+        "samples" => samples.size,
+        "passed" => passed_count(per_sample_scores.slice(*samples.map(&:id))),
+        "provider" => providers.tally.max_by { |_provider, count| count }&.first,
+        "avg_duration_ms" => durations_ms.any? ? (durations_ms.sum / durations_ms.size).round : nil,
+        "input_tokens" => input_tokens,
+        "output_tokens" => output_tokens,
+        "cost" => costs.any? ? costs.sum.round(6) : nil
+      }
+    end
+
+    # --- Judge usage -----------------------------------------------------------
+    #
+    # Every call the judge makes is metered here, apart from the agent's own
+    # spend: scoring answers, recommending fixes, writing the verdict and,
+    # for a judge_defined evaluation, authoring the KPIs. The agent's cost is
+    # the operating figure — what serving these interactions costs — while
+    # the judge's is the evaluation's own, offline, agent-to-agent overhead,
+    # and a run that reported the two as one number would overstate the
+    # first. Persisted as scores["_judge_usage"]:
+    #
+    #   { "calls", "input_tokens", "output_tokens", "cost", "model",
+    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n } }
+    #
+    # nil until the judge has been asked something, so a rules-only run
+    # records no judge at all rather than a judge that cost nothing.
+
+    def judge_usage
+      return nil if @judge_usage.nil?
+
+      @judge_usage.merge("cost" => @judge_usage["cost"]&.round(6))
+    end
+
+    # Asks the judge and meters the answer. `kind` is the call's purpose —
+    # :score, :recommend, :verdict or :define — the same vocabulary
+    # ActiveAgent::Evals::Judge hands a block that accepts `kind:`.
+    def judge_generate(kind, message:, instructions:)
+      response = judge_class.prompt(message: message, instructions: instructions).generate_now
+      record_judge_call(kind, response)
+      response
+    end
+
+    def record_judge_call(kind, response)
+      usage = response.respond_to?(:usage) ? response.usage : nil
+      input_tokens = usage.respond_to?(:input_tokens) ? usage.input_tokens.to_i : 0
+      output_tokens = usage.respond_to?(:output_tokens) ? usage.output_tokens.to_i : 0
+      model = (response.respond_to?(:model) && response.model.presence) || @evaluation.judge_model.presence
+      cost = ModelPricing.estimate(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
+
+      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {} }
+      @judge_usage["calls"] += 1
+      @judge_usage["input_tokens"] += input_tokens
+      @judge_usage["output_tokens"] += output_tokens
+      @judge_usage["cost"] = (@judge_usage["cost"] || 0.0) + cost if cost
+      @judge_usage["model"] ||= model
+      @judge_usage["by_kind"][kind.to_s] = @judge_usage["by_kind"].fetch(kind.to_s, 0) + 1
+    end
+
     def sample_generations(model: nil)
       scope = @evaluation.agent.generations
       scope = scope.where(model: model) if model
@@ -175,7 +275,7 @@ module ActionAgent
       if total.zero?
         return {
           "skipped" => true,
-          "reason" => "No telemetry traces for #{@evaluation.agent.telemetry_agent_class} in the last #{window_hours}h"
+          "reason" => "No telemetry traces for #{telemetry_source} in the last #{window_hours}h"
         }
       end
 
@@ -213,12 +313,18 @@ module ActionAgent
     end
 
     def telemetry_traces(window_hours)
-      ActionAgent.trace_model
-        .for_account(ActionAgent.tenant_for(owner))
-        .for_agent(@evaluation.agent.telemetry_agent_class)
+      @evaluation.agent
+        .telemetry_traces(ActionAgent.trace_model.for_account(ActionAgent.tenant_for(owner)))
         .for_date_range(window_hours.hours.ago, Time.current)
     end
 
+    # What a skip reason says was looked for: the observed agent itself, or
+    # the class any other agent's traces are reported under.
+    def telemetry_source
+      agent = @evaluation.agent
+      agent.observed? ? agent.name : agent.telemetry_agent_class
+    end
+
     # Returns 0.0..1.0, or nil when the criterion cannot be scored.
     def score_sample(criterion, generation)
       config = criterion["config"] || {}
@@ -273,10 +379,11 @@ module ActionAgent
         raise "Judge-defined KPIs need provider credentials (add a provider API key in Settings)"
       end
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :define,
         message: kpi_definition_prompt,
         instructions: "You define measurable evaluation KPIs for AI agents. Respond ONLY with JSON."
-      ).generate_now
+      )
 
       kpis = parse_kpis(response.message&.content)
       raise "Judge returned no usable KPIs — try again or add criteria manually" if kpis.empty?
@@ -354,7 +461,8 @@ module ActionAgent
         "#{key}: #{cells.join(', ')}"
       end
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :verdict,
         message: <<~PROMPT,
           An AI agent was evaluated under multiple models. Its goals:
           ---
@@ -368,7 +476,7 @@ module ActionAgent
           Respond ONLY with JSON: {"winner": "<model>", "rationale": "<at most two sentences>"}
         PROMPT
         instructions: "You are an impartial evaluation judge comparing model cohorts. Respond ONLY with JSON."
-      ).generate_now
+      )
 
       json = response.message&.content.to_s[/\{.*\}/m]
       verdict = json ? JSON.parse(json) : nil
@@ -390,10 +498,11 @@ module ActionAgent
       return nil unless judge_available?
       return nil if generation.content.blank?
 
-      response = judge_class.prompt(
+      response = judge_generate(
+        :score,
         message: judge_prompt(criterion, generation),
         instructions: "You are an impartial evaluation judge. Respond ONLY with JSON: {\"score\": <float between 0.0 and 1.0>}"
-      ).generate_now
+      )
 
       parse_judge_score(response.message&.content)
     rescue StandardError => e

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -261,6 +261,10 @@ module ActionAgent
       scores["_selection"] = run.selection
       scores["_metadata"] = report.metadata
       scores["_judge_label"] = report.judge_label || report.judge&.label
+      # The judge's own spend, apart from the replays' (which the results
+      # carry and EvaluationRun#usage sums). A host adapter that runs its own
+      # judge is out of reach of this meter, so its runs record none.
+      scores["_judge_usage"] = judge_usage if judge_usage
       scores
     end
 
@@ -318,12 +322,13 @@ module ActionAgent
 
     # The judge the evaluation's owner has credentials for, wrapped for the
     # evaluation core; nil when none is configured, in which case scoring
-    # stays on rules and expectations.
+    # stays on rules and expectations. The block takes `kind:` so each call
+    # is metered under what it was for (EvaluationRunnerService#judge_generate).
     def evals_judge
       return nil unless judge_available?
 
-      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:|
-        judge_class.prompt(message: prompt, instructions: instructions).generate_now.message&.content
+      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:, kind:|
+        judge_generate(kind, message: prompt, instructions: instructions).message&.content
       end
     end
   end

data/config/routes.rb

@@ -27,6 +27,11 @@ ActionAgent::Engine.routes.draw do
     # Authenticated with a bearer token, not a session.
     resources :traces, only: [ :create ]
 
+    # The collector for evaluation reports an application ran itself
+    # (ActiveAgent::Evals::Publisher), at <mount>/api/evaluation_reports.
+    # Authenticated like trace ingest, with a bearer token.
+    resources :evaluation_reports, only: [ :create ]
+
     # A JSON API has no :new or :edit forms to serve.
     resources :agents, except: [ :new, :edit ] do
       member do

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.6.4"
+  VERSION = "1.7.0"
 end

data/lib/action_agent.rb

@@ -209,24 +209,77 @@ module ActionAgent
     # @return [Object, nil] Object responding to #signed_url_for and #fetch_snapshot
     attr_accessor :storage_service
 
-    # Bearer token required by the ingest API in single-tenant mode. When
-    # unset the local ingest endpoint accepts unauthenticated posts, so set
-    # it whenever the mount is reachable beyond your own machine.
-    # (Multi-tenant mode authenticates per-account keys instead.)
+    # Bearer token required in single-tenant mode by the endpoints other
+    # applications post to: trace ingest (<mount>/api/traces) and published
+    # evaluation reports (<mount>/api/evaluation_reports). When unset both
+    # accept unauthenticated posts, so set it whenever the mount is reachable
+    # beyond your own machine. Trace ingest also takes a form post, which any
+    # web page open in a browser on that machine can send it; the report
+    # collector takes only application/json, which a page cannot send
+    # cross-site. (Multi-tenant mode authenticates per-account keys instead.)
     # @return [String, nil]
     attr_accessor :ingest_api_key
 
+    # Concerns included into ActionAgent::ApplicationRecord as it loads, and
+    # through it into every engine model. An entry is a Module or the name
+    # of one. A name is resolved when the class
+    # loads, so an initializer can refer to a constant the host has not
+    # autoloaded yet, and a name that resolves to nothing raises NameError
+    # there rather than being skipped.
+    #
+    #   ActionAgent.configure do |config|
+    #     config.model_concerns = ["MyApp::ConnectionSwitching"]
+    #   end
+    #
+    # The class loads after the initializers have run, so set this in an
+    # initializer; a concern added later is not applied.
+    # @return [Array<Module, String>]
+    attr_accessor :model_concerns
+
+    # Concerns included into ActionAgent::ApplicationController as it loads,
+    # and through it into every dashboard controller: the React dashboard
+    # and its JSON API, the server-rendered console and the MCP facade.
+    # They are included ahead of the engine's own callbacks, so a concern's
+    # before_action or around_action runs before the dashboard
+    # authenticates. Entries are Modules or names, as for model_concerns.
+    #
+    # Not the endpoints other applications post to: Api::TracesController
+    # and Api::EvaluationReportsController authenticate with a bearer token
+    # and inherit ActionController::API.
+    #
+    #   ActionAgent.configure do |config|
+    #     config.controller_concerns = ["MyApp::RequestTagging"]
+    #   end
+    # @return [Array<Module, String>]
+    attr_accessor :controller_concerns
+
     # @deprecated Never consumed — dashboard controllers inherit
-    #   ActionController::Base. Retained as a no-op so existing
-    #   initializers that set it keep booting; remove in the next major.
+    #   ActionController::Base, and controller_concerns is how a host puts
+    #   its own behaviour on them. Assigning it warns and stores a value
+    #   nothing reads; removed in the next major.
     # @return [String]
-    attr_accessor :base_controller_class
+    attr_reader :base_controller_class
+
+    def base_controller_class=(value)
+      deprecator.warn(
+        "ActionAgent.base_controller_class has never been consumed and is removed in 2.0. " \
+        "Set ActionAgent.controller_concerns to extend the dashboard's controllers."
+      )
+      @base_controller_class = value
+    end
 
-    # Called before each run/trace-ingest to enforce host-app limits.
-    # Receives (owner, kind) where kind is :execution or :trace_ingest, and
-    # returns nil to allow, or to deny: a message String, or a Hash merged
-    # into the response so the app can surface its own usage numbers.
-    # Denials surface as HTTP 402 (execution) / 429 (ingest).
+    # Called before each metered action to enforce host-app limits.
+    # Receives (owner, kind) and returns nil to allow, or to deny: a message
+    # String, or a Hash merged into the response so the app can surface its
+    # own usage numbers. The kinds, and how a denial surfaces:
+    #
+    #   :execution         — an agent run; HTTP 402
+    #   :trace_ingest      — a POST to <mount>/api/traces; HTTP 429
+    #   :evaluation_report — a report <mount>/api/evaluation_reports would
+    #                        store (never an identical retry); HTTP 429
+    #
+    # The owner of an ingest kind is the tenant the key resolved to, nil on a
+    # single-tenant install.
     #
     # Unset means unlimited, which is what a self-hosted install wants.
     # @return [Proc, nil]
@@ -256,6 +309,21 @@ module ActionAgent
     # @return [Boolean]
     attr_accessor :execution_enabled
 
+    # Whether a run of an agent that mirrors a host class executes that class,
+    # instead of the class the engine builds from the record's `tools` and
+    # `instructions` columns.
+    #
+    # Off by default: it changes what a run of a mirrored agent executes, and
+    # a host that has tuned its dashboard records around the dynamic runtime
+    # should opt in deliberately. Dashboard-authored agents — the ones with no
+    # `agent_class_name` — are unaffected either way.
+    #
+    # On, a mirrored agent runs its real tools, delegations and instructions,
+    # so an evaluation scores the agent production runs rather than a
+    # flattened copy of it.
+    # @return [Boolean]
+    attr_accessor :run_host_agent_classes
+
     # Whether the "Ask ActiveAgents" assistant is available.
     #
     # The assistant is a tool for developing and CI-ing agents: it sends
@@ -314,7 +382,9 @@ module ActionAgent
 
     # Called after the dashboard performs a metered action, as
     # (owner, kind) — the counterpart to quota_checker, for host apps that
-    # track usage against a plan. Unset means nothing is counted.
+    # track usage against a plan. The kinds are :execution, for each agent
+    # run, and :evaluation_report, for each report the collector stores; an
+    # identical retry is not counted again. Unset means nothing is counted.
     # @return [Proc, nil]
     attr_accessor :usage_recorder
 
@@ -323,6 +393,11 @@ module ActionAgent
     # nobody in single-tenant mode. A host app whose agents hang off a
     # different record (the platform's hang off the account's owning user)
     # supplies its own mapping.
+    #
+    # A published evaluation report's agent is placed the same way: the
+    # resolver receives an unsaved trace with the publishing tenant as its
+    # account, and the report's source and agent name as its service_name and
+    # agent_class. In multi-tenant mode it must not return nil there.
     # @return [Proc, nil]
     attr_accessor :trace_owner_resolver
 
@@ -513,6 +588,19 @@ module ActionAgent
       name&.safe_constantize
     end
 
+    # The modules model_concerns names. ApplicationRecord reads it as it loads.
+    # @return [Array<Module>]
+    def model_concern_modules
+      resolve_concerns(model_concerns)
+    end
+
+    # The modules controller_concerns names. ApplicationController reads it
+    # as it loads.
+    # @return [Array<Module>]
+    def controller_concern_modules
+      resolve_concerns(controller_concerns)
+    end
+
     # Configures the dashboard.
     #
     # @yield [config] Configuration block
@@ -540,10 +628,13 @@ module ActionAgent
       @storage_service = nil
       @ingest_api_key = nil
       @base_controller_class = "ActionController::Base" # deprecated no-op
+      @model_concerns = []
+      @controller_concerns = []
       @quota_checker = nil
       @provider_credentials_resolver = nil
       @sandbox_backends = {}
       @execution_enabled = true
+      @run_host_agent_classes = false
       @assistant_enabled = nil
 
       @scenario_evaluation_adapter_resolver = nil
@@ -639,6 +730,12 @@ module ActionAgent
     def schema_tool_class_for(name)
       schema_tool_classes.find { |klass| klass.tool?(name) }
     end
+
+    private
+
+    def resolve_concerns(entries)
+      Array(entries).map { |entry| entry.is_a?(Module) ? entry : entry.to_s.constantize }
+    end
   end
 
   # Set defaults

data/lib/generators/action_agent/install_generator.rb

@@ -77,13 +77,36 @@ module ActionAgent
         )
       end
 
+      # add_agent_releases is emitted before the dashboard tables, so on an
+      # install generated fresh before the create-table migration carried the
+      # release columns it skipped every table, and its copy may name tables
+      # without the configured prefix. This adds whatever is still missing,
+      # and changes nothing where the columns exist.
+      unless existing_migration?("ensure_agent_release_columns")
+        migration_template(
+          "ensure_agent_release_columns.rb.erb",
+          "db/migrate/ensure_agent_release_columns.rb"
+        )
+      end
+
       # Scenario suites arrived after the dashboard tables shipped, so an
       # install that already has those still needs this one.
-      return if existing_migration?("create_active_agent_evaluation_scenarios")
+      unless existing_migration?("create_active_agent_evaluation_scenarios")
+        migration_template(
+          "create_active_agent_evaluation_scenarios.rb.erb",
+          "db/migrate/create_active_agent_evaluation_scenarios.rb"
+        )
+      end
+
+      # Published evaluation reports arrived after the dashboard tables
+      # shipped. Emitted after them, so on a fresh install it runs once the
+      # table exists, finds the columns the create-table migration made, and
+      # changes nothing.
+      return if existing_migration?("add_evaluation_report_identity")
 
       migration_template(
-        "create_active_agent_evaluation_scenarios.rb.erb",
-        "db/migrate/create_active_agent_evaluation_scenarios.rb"
+        "add_evaluation_report_identity.rb.erb",
+        "db/migrate/add_evaluation_report_identity.rb"
       )
     end
 

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -44,9 +44,11 @@ ActionAgent.configure do |config|
   # Ingest API authentication
   # ==========================================================================
   #
-  # Bearer token other apps must send when posting traces to the mounted
-  # ingest endpoint (/activeagents/api/traces). Leave unset only when the
-  # mount is not reachable beyond your own machine.
+  # Bearer token other apps must send when posting traces
+  # (/activeagents/api/traces) or evaluation reports
+  # (/activeagents/api/evaluation_reports) to the mount. Leave unset only
+  # when the mount is not reachable beyond your own machine; even then, any web
+  # page open in a browser there can post form data to the trace endpoint.
   #
   # config.ingest_api_key = Rails.application.credentials.dig(:active_agent, :ingest_api_key)
 
@@ -78,6 +80,20 @@ ActionAgent.configure do |config|
   # nothing rather than everything. An empty dashboard for a signed-in user
   # means the resolver above returned nil.
 
+  # ==========================================================================
+  # Host integration
+  # ==========================================================================
+  #
+  # Concerns your own models and controllers carry, applied to the engine's.
+  # model_concerns are included into ActionAgent::ApplicationRecord, and so
+  # into every engine model. controller_concerns are included into
+  # ActionAgent::ApplicationController ahead of its own callbacks, and so
+  # into every dashboard controller — not the bearer-token ingest endpoints.
+  # Modules or their names; a name is resolved when the class loads.
+  #
+  # config.model_concerns = ["ConnectionSwitching"]
+  # config.controller_concerns = ["RequestTagging"]
+
   # ==========================================================================
   # Ask ActiveAgents assistant
   # ==========================================================================

data/lib/generators/action_agent/templates/add_agent_releases.rb.erb

@@ -7,33 +7,38 @@
 # migration carries the same columns for a fresh install.
 #
 # Each step is guarded so the migration is safe to re-run against a database
-# that already has some of these columns.
+# that already has some of these columns. Table names follow
+# ActionAgent.table_name_prefix, except the trace table, whose name is fixed.
 class AddAgentReleases < ActiveRecord::Migration<%= migration_version %>
   def up
-    add_column_unless_exists :active_agent_agents, :release_digest, :string
+    add_column_unless_exists table(:agents), :release_digest, :string
 
-    add_column_unless_exists :active_agent_agent_versions, :release_digest, :string
-    add_column_unless_exists :active_agent_agent_versions, :revision, :string
-    add_index_unless_exists :active_agent_agent_versions, [ :agent_id, :release_digest ]
+    add_column_unless_exists table(:agent_versions), :release_digest, :string
+    add_column_unless_exists table(:agent_versions), :revision, :string
+    add_index_unless_exists table(:agent_versions), [ :agent_id, :release_digest ]
 
-    %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table|
-      add_column_unless_exists table, :agent_version_id, :bigint
-      add_index_unless_exists table, :agent_version_id
+    [ "active_agent_telemetry_traces", table(:agent_runs), table(:evaluation_runs) ].each do |name|
+      add_column_unless_exists name, :agent_version_id, :bigint
+      add_index_unless_exists name, :agent_version_id
     end
   end
 
   def down
-    remove_column :active_agent_agents, :release_digest if column_exists?(:active_agent_agents, :release_digest)
-    remove_column :active_agent_agent_versions, :release_digest if column_exists?(:active_agent_agent_versions, :release_digest)
-    remove_column :active_agent_agent_versions, :revision if column_exists?(:active_agent_agent_versions, :revision)
+    remove_column table(:agents), :release_digest if column_exists?(table(:agents), :release_digest)
+    remove_column table(:agent_versions), :release_digest if column_exists?(table(:agent_versions), :release_digest)
+    remove_column table(:agent_versions), :revision if column_exists?(table(:agent_versions), :revision)
 
-    %i[active_agent_telemetry_traces active_agent_agent_runs active_agent_evaluation_runs].each do |table|
-      remove_column table, :agent_version_id if column_exists?(table, :agent_version_id)
+    [ "active_agent_telemetry_traces", table(:agent_runs), table(:evaluation_runs) ].each do |name|
+      remove_column name, :agent_version_id if column_exists?(name, :agent_version_id)
     end
   end
 
   private
 
+  def table(name)
+    "#{ActionAgent.table_name_prefix}#{name}"
+  end
+
   def add_column_unless_exists(table, column, type)
     return unless table_exists?(table)
     return if column_exists?(table, column)

data/lib/generators/action_agent/templates/add_evaluation_report_identity.rb.erb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Published evaluation reports: an application that runs its agents itself
+# can evaluate them in-process and POST the finished report to
+# <mount>/api/evaluation_reports (ActionAgent::EvaluationReportImport), which
+# stores it as an evaluation run. The run is identified by the run_id the
+# application minted, within the tenant whose key published it, and carries
+# the digest of the report, so an identical retry resolves to the same run and
+# different content under the same run_id is refused.
+#
+# The tenant is its id, or "" on a single-tenant install, because a unique
+# index never treats two NULLs as equal. Both are compared exactly, which on
+# MySQL takes a binary collation. All three columns are NULL for a run the
+# dashboard executed. Emitted for an install whose dashboard tables predate
+# published reports; the create-table migration carries the same columns for
+# a fresh install. Each step is guarded, so the migration is safe to re-run.
+class AddEvaluationReportIdentity < ActiveRecord::Migration<%= migration_version %>
+  IDENTITY = [ :external_tenant, :external_run_id ].freeze
+
+  def up
+    return unless table_exists?(table)
+
+    IDENTITY.each do |column|
+      add_column table, column, :string, **exact_collation unless column_exists?(table, column)
+    end
+    add_column table, :external_report_digest, :string unless column_exists?(table, :external_report_digest)
+    add_index table, IDENTITY, unique: true, name: index_name unless index_exists?(table, IDENTITY, name: index_name)
+  end
+
+  def down
+    return unless table_exists?(table)
+
+    remove_index table, name: index_name if index_exists?(table, IDENTITY, name: index_name)
+    %i[external_tenant external_run_id external_report_digest].each do |column|
+      remove_column table, column if column_exists?(table, column)
+    end
+  end
+
+  private
+
+  def table
+    "#{ActionAgent.table_name_prefix}evaluation_runs"
+  end
+
+  def index_name
+    "index_#{ActionAgent.table_name_prefix}evaluation_runs_on_external_identity"
+  end
+
+  # MySQL's default collation ignores case and accents, so "nightly-a" and
+  # "Nightly-A" would be one run_id there and two everywhere else.
+  def exact_collation
+    connection.adapter_name.to_s.downcase.match?(/mysql|trilogy/) ? { collation: "utf8mb4_bin" } : {}
+  end
+end