actionagent 1.8.0 → 1.9.0

data/app/services/action_agent/evaluation_runner_service.rb

@@ -190,7 +190,9 @@ module ActionAgent
     # The cost here is the agent's: what the sampled interactions cost to
     # operate. It is not what this run spent — scoring recorded generations
     # costs nothing until a judge is asked, and the judge's own spend is
-    # recorded apart, under "_judge_usage" (see #judge_usage).
+    # recorded apart, under "_judge_usage" (see #judge_usage). It sums the
+    # generations that could be priced, and "priced" counts them, so a
+    # cohort with some unpriced generations reads as partial.
 
     def cohort_summaries(samples, per_sample_scores)
       samples
@@ -207,7 +209,8 @@ module ActionAgent
       input_tokens = samples.sum { |generation| generation.input_tokens.to_i }
       output_tokens = samples.sum { |generation| generation.output_tokens.to_i }
       costs = samples.filter_map do |generation|
-        ModelPricing.estimate(model: generation.model, input_tokens: generation.input_tokens, output_tokens: generation.output_tokens)
+        ModelPricing.estimate(model: generation.model, provider: generation.provider.presence, input_tokens: generation.input_tokens,
+                              output_tokens: generation.output_tokens)
       end
 
       {
@@ -217,7 +220,8 @@ module ActionAgent
         "avg_duration_ms" => durations_ms.any? ? (durations_ms.sum / durations_ms.size).round : nil,
         "input_tokens" => input_tokens,
         "output_tokens" => output_tokens,
-        "cost" => costs.any? ? costs.sum.round(6) : nil
+        "cost" => costs.any? ? costs.sum.round(6) : nil,
+        "priced" => costs.size
       }
     end
 
@@ -254,10 +258,10 @@ module ActionAgent
 
     def record_judge_call(kind, response)
       usage = response.respond_to?(:usage) ? response.usage : nil
-      input_tokens = usage.respond_to?(:input_tokens) ? usage.input_tokens.to_i : 0
+      input_tokens = judge_input_tokens(usage)
       output_tokens = usage.respond_to?(:output_tokens) ? usage.output_tokens.to_i : 0
       model = (response.respond_to?(:model) && response.model.presence) || @evaluation.judge_model.presence
-      cost = ModelPricing.estimate(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
+      cost = ModelPricing.estimate(model: model, provider: judge_provider, input_tokens: input_tokens, output_tokens: output_tokens)
 
       @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {} }
       @judge_usage["calls"] += 1
@@ -268,6 +272,22 @@ module ActionAgent
       @judge_usage["by_kind"][kind.to_s] = @judge_usage["by_kind"].fetch(kind.to_s, 0) + 1
     end
 
+    # Every token the judge was billed for reading. Anthropic reports the
+    # tokens read from or written to the prompt cache apart from
+    # `input_tokens`, so they are added; OpenAI's prompt count already holds
+    # its cached tokens. Priced at the input rate, which overstates a cache
+    # read: the figure is an upper bound.
+    def judge_input_tokens(usage)
+      return 0 unless usage.respond_to?(:input_tokens)
+
+      total = usage.input_tokens.to_i
+      return total unless judge_provider == :anthropic
+
+      total += usage.cached_tokens.to_i if usage.respond_to?(:cached_tokens)
+      total += usage.cache_creation_tokens.to_i if usage.respond_to?(:cache_creation_tokens)
+      total
+    end
+
     def sample_generations(model: nil)
       scope = @evaluation.agent.generations
       scope = scope.where(model: model) if model
@@ -574,13 +594,12 @@ module ActionAgent
 
     # The evaluated agent owner's credential for +name+ (Settings ->
     # Provider API Keys, or whatever the host app resolves); preferred over
-    # the host's config/active_agent.yml credentials for the judge.
+    # the host's config/active_agent.yml credentials for the judge. Resolved
+    # with no actor: the judge scores for the organization, so it never runs
+    # on a member's personal key.
     def owner_provider_options(name)
       @owner_provider_options ||= {}
-      @owner_provider_options[name.to_s] ||= begin
-        from_host = ActionAgent.provider_credentials(owner, name.to_s)
-        from_host.presence || ProviderKey.for_owner(owner).find_by(provider: name.to_s)&.generation_options || {}
-      end
+      @owner_provider_options[name.to_s] ||= ProviderCredentials.resolve(owner: owner, actor: nil, provider: name).options
     end
 
     def owner

data/app/services/action_agent/evaluation_standing.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Where an evaluation stands against the agent as it is now, so a pass
+  # rate on the Evaluations page describes the current agent rather than
+  # pooling every suite's last run, however old the code it scored.
+  #
+  # The evaluation's headline run is its newest complete run; a newer run
+  # still pending or failed shows beside it, never in its place. Its
+  # standing is one of:
+  #
+  #   current    — the headline run scored the agent's current version
+  #   stale      — it scored an earlier version, or a release the dashboard
+  #                has not matched to the agent's current release
+  #   unrecorded — the run recorded no version and the agent has no release
+  #                versions at all, so nothing says which code it scored
+  #   archived   — the evaluation is archived (Evaluation#archive!)
+  #   none       — no complete run
+  #
+  # Only edits that change how a run goes count as a new version: what the
+  # model is given (the instructions, the action prompts, the tools, the MCP
+  # servers, the model config and the response format), and the tools whose
+  # calls wait for a person's approval, which pause a replay. A dashboard
+  # edit to the agent's appearance cuts an AgentVersion like any other edit,
+  # but the run before it scored the same agent and stays current.
+  class EvaluationStanding
+    STANDINGS = %w[current stale unrecorded archived none].freeze
+    VERSION_STATES = %w[current earlier unrecorded].freeze
+    # The configuration snapshot keys the model is given.
+    MODEL_FACING = %w[instructions action_prompts tools mcp_servers model_config response_format].freeze
+    # The configuration snapshot keys that change how a run goes.
+    SCORED = (MODEL_FACING + %w[approval_required_tools]).freeze
+
+    # Gives every evaluation its standing with one query per table rather
+    # than one per evaluation: the agents' latest versions and whether each
+    # has a release are read for the page at once.
+    def self.preload(evaluations)
+      evaluations = Array(evaluations)
+      agent_ids = evaluations.filter_map(&:agent_id).uniq
+      return if agent_ids.empty?
+
+      newest = AgentVersion.where(agent_id: agent_ids).group(:agent_id).maximum(:version_number)
+      latest = AgentVersion.where(agent_id: agent_ids).where(version_number: newest.values.uniq)
+        .select { |version| newest[version.agent_id] == version.version_number }.index_by(&:agent_id)
+      releases = AgentVersion.releases.where(agent_id: agent_ids).distinct.pluck(:agent_id).to_set
+
+      evaluations.each do |evaluation|
+        evaluation.standing_info = new(evaluation, latest_version: latest[evaluation.agent_id], has_releases: releases.include?(evaluation.agent_id))
+      end
+    end
+
+    attr_reader :evaluation
+
+    # @param headline_run [EvaluationRun, nil] the headline run when the
+    #   caller already holds it (AgentScorecard selects one per evaluation)
+    def initialize(evaluation, latest_version: :unknown, has_releases: :unknown, headline_run: :unknown)
+      @evaluation = evaluation
+      @latest_version = latest_version
+      @has_releases = has_releases
+      @headline_run = headline_run
+    end
+
+    def headline_run
+      return @headline_run unless @headline_run == :unknown
+
+      @headline_run = evaluation.headline_run
+    end
+
+    # The same standing read against +run+ as the headline run.
+    def with_headline(run)
+      self.class.new(evaluation, latest_version: latest_version, has_releases: has_releases?, headline_run: run)
+    end
+
+    def standing
+      return "archived" if evaluation.archived?
+      return "none" unless headline_run
+
+      case version_state(headline_run)
+      when "current" then "current"
+      when "unrecorded" then has_releases? ? "stale" : "unrecorded"
+      else "stale"
+      end
+    end
+
+    # Whether +run+ scored the agent's current version: "current",
+    # "earlier", or "unrecorded" when it recorded no version.
+    def version_state(run)
+      version = run&.agent_version
+      return "unrecorded" unless version
+
+      # The agent's last recorded deploy is the truth about the code that
+      # runs: a release version matching it is current whatever the
+      # dashboard edited since, and one that does not is earlier even when
+      # it was recorded last (a report of an older deploy published late).
+      if version.release? && (deployed = evaluation.agent&.release_digest.presence)
+        return deployed == version.release_digest ? "current" : "earlier"
+      end
+
+      latest = latest_version
+      return "current" if latest.nil? || latest.id == version.id
+      return "current" if scored_configuration(version) == scored_configuration(latest)
+
+      "earlier"
+    end
+
+    # Passes per model label of the headline run, `{ label => { passed, total } }`,
+    # from its recorded summaries; empty without them.
+    def per_model
+      run = headline_run
+      return {} unless run
+
+      summaries = run.scores.is_a?(Hash) ? (run.scores["_models"].presence || run.scores["_cohorts"]) : nil
+      return {} unless summaries.is_a?(Hash)
+
+      summaries.filter_map do |label, stats|
+        next unless stats.is_a?(Hash)
+
+        total = (stats["scenarios"] || stats["samples"]).to_i
+        [ label, { passed: stats["passed"].to_i, total: total } ]
+      end.to_h
+    end
+
+    # What the index and the scorecards pool: a current or unrecorded
+    # headline run; a stale or archived evaluation is left out and said so.
+    def counted?
+      %w[current unrecorded].include?(standing)
+    end
+
+    private
+
+    def latest_version
+      return @latest_version unless @latest_version == :unknown
+
+      @latest_version = evaluation.agent&.latest_version
+    end
+
+    def has_releases?
+      return @has_releases unless @has_releases == :unknown
+
+      @has_releases = evaluation.agent&.agent_versions&.releases&.exists? || false
+    end
+
+    # A snapshot taken before an agent had an approval list reads as an empty
+    # one, so the list's arrival alone cuts no new version.
+    def scored_configuration(version)
+      snapshot = version.configuration_snapshot.to_h.stringify_keys
+      configuration = SCORED.to_h { |key| [ key, snapshot[key] ] }
+      configuration["approval_required_tools"] = Array(configuration["approval_required_tools"])
+      configuration
+    end
+  end
+end