actionagent 1.7.2 → 1.8.1

data/app/services/action_agent/evaluation_standing.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Where an evaluation stands against the agent as it is now, so a pass
+  # rate on the Evaluations page describes the current agent rather than
+  # pooling every suite's last run, however old the code it scored.
+  #
+  # The evaluation's headline run is its newest complete run; a newer run
+  # still pending or failed shows beside it, never in its place. Its
+  # standing is one of:
+  #
+  #   current    — the headline run scored the agent's current version
+  #   stale      — it scored an earlier version, or a release the dashboard
+  #                has not matched to the agent's current release
+  #   unrecorded — the run recorded no version and the agent has no release
+  #                versions at all, so nothing says which code it scored
+  #   archived   — the evaluation is archived (Evaluation#archive!)
+  #   none       — no complete run
+  #
+  # Only edits the model can see count as a new version: the instructions,
+  # the action prompts, the tools, the MCP servers, the model config and
+  # the response format. A dashboard edit to the agent's appearance cuts an
+  # AgentVersion like any other edit, but the run before it scored the same
+  # agent and stays current.
+  class EvaluationStanding
+    STANDINGS = %w[current stale unrecorded archived none].freeze
+    VERSION_STATES = %w[current earlier unrecorded].freeze
+    # The configuration snapshot keys the model is given.
+    MODEL_FACING = %w[instructions action_prompts tools mcp_servers model_config response_format].freeze
+
+    # Gives every evaluation its standing with one query per table rather
+    # than one per evaluation: the agents' latest versions and whether each
+    # has a release are read for the page at once.
+    def self.preload(evaluations)
+      evaluations = Array(evaluations)
+      agent_ids = evaluations.filter_map(&:agent_id).uniq
+      return if agent_ids.empty?
+
+      newest = AgentVersion.where(agent_id: agent_ids).group(:agent_id).maximum(:version_number)
+      latest = AgentVersion.where(agent_id: agent_ids).where(version_number: newest.values.uniq)
+        .select { |version| newest[version.agent_id] == version.version_number }.index_by(&:agent_id)
+      releases = AgentVersion.releases.where(agent_id: agent_ids).distinct.pluck(:agent_id).to_set
+
+      evaluations.each do |evaluation|
+        evaluation.standing_info = new(evaluation, latest_version: latest[evaluation.agent_id], has_releases: releases.include?(evaluation.agent_id))
+      end
+    end
+
+    attr_reader :evaluation
+
+    # @param headline_run [EvaluationRun, nil] the headline run when the
+    #   caller already holds it (AgentScorecard selects one per evaluation)
+    def initialize(evaluation, latest_version: :unknown, has_releases: :unknown, headline_run: :unknown)
+      @evaluation = evaluation
+      @latest_version = latest_version
+      @has_releases = has_releases
+      @headline_run = headline_run
+    end
+
+    def headline_run
+      return @headline_run unless @headline_run == :unknown
+
+      @headline_run = evaluation.headline_run
+    end
+
+    # The same standing read against +run+ as the headline run.
+    def with_headline(run)
+      self.class.new(evaluation, latest_version: latest_version, has_releases: has_releases?, headline_run: run)
+    end
+
+    def standing
+      return "archived" if evaluation.archived?
+      return "none" unless headline_run
+
+      case version_state(headline_run)
+      when "current" then "current"
+      when "unrecorded" then has_releases? ? "stale" : "unrecorded"
+      else "stale"
+      end
+    end
+
+    # Whether +run+ scored the agent's current version: "current",
+    # "earlier", or "unrecorded" when it recorded no version.
+    def version_state(run)
+      version = run&.agent_version
+      return "unrecorded" unless version
+
+      # The agent's last recorded deploy is the truth about the code that
+      # runs: a release version matching it is current whatever the
+      # dashboard edited since, and one that does not is earlier even when
+      # it was recorded last (a report of an older deploy published late).
+      if version.release? && (deployed = evaluation.agent&.release_digest.presence)
+        return deployed == version.release_digest ? "current" : "earlier"
+      end
+
+      latest = latest_version
+      return "current" if latest.nil? || latest.id == version.id
+      return "current" if model_facing(version) == model_facing(latest)
+
+      "earlier"
+    end
+
+    # Passes per model label of the headline run, `{ label => { passed, total } }`,
+    # from its recorded summaries; empty without them.
+    def per_model
+      run = headline_run
+      return {} unless run
+
+      summaries = run.scores.is_a?(Hash) ? (run.scores["_models"].presence || run.scores["_cohorts"]) : nil
+      return {} unless summaries.is_a?(Hash)
+
+      summaries.filter_map do |label, stats|
+        next unless stats.is_a?(Hash)
+
+        total = (stats["scenarios"] || stats["samples"]).to_i
+        [ label, { passed: stats["passed"].to_i, total: total } ]
+      end.to_h
+    end
+
+    # What the index and the scorecards pool: a current or unrecorded
+    # headline run; a stale or archived evaluation is left out and said so.
+    def counted?
+      %w[current unrecorded].include?(standing)
+    end
+
+    private
+
+    def latest_version
+      return @latest_version unless @latest_version == :unknown
+
+      @latest_version = evaluation.agent&.latest_version
+    end
+
+    def has_releases?
+      return @has_releases unless @has_releases == :unknown
+
+      @has_releases = evaluation.agent&.agent_versions&.releases&.exists? || false
+    end
+
+    def model_facing(version)
+      snapshot = version.configuration_snapshot.to_h.stringify_keys
+      MODEL_FACING.to_h { |key| [ key, snapshot[key] ] }
+    end
+  end
+end

data/app/services/action_agent/evaluation_tool_resolver.rb

@@ -80,6 +80,24 @@ module ActionAgent
       configured_keys.to_a
     end
 
+    # The tools the agent's entry for +key+ allows, or nil when that entry
+    # names none — which offers every tool the server serves. This is the rule
+    # AgentToolRoster#allowed_tools reads for the Tools tab, which saves
+    # {key, name, tools: [...]} when some are switched off and tools: [] when
+    # all are: a later entry for the same key wins, as it does there.
+    #
+    # @param key [String] a server key
+    # @return [Array<String>, nil]
+    def allowed_tools_for(key)
+      entry = configured_entries.select { |candidate| normalize(entry_key(candidate)) == normalize(key) }.last
+      return nil unless entry.respond_to?(:key?)
+
+      names = entry["tools"] || entry[:tools]
+      return nil if names.nil?
+
+      Array(names).filter_map { |tool| (tool.respond_to?(:key?) ? tool["name"] || tool[:name] : tool).to_s.strip.presence }
+    end
+
     private
 
     # The catalog's name when it has one; otherwise the name the agent's

data/app/services/action_agent/github_client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # The few GitHub calls the dashboard makes: the OAuth code exchange, the
+  # authenticated user, and the repositories that user can reach. Plain
+  # Net::HTTP, like the provider model lookups, so the engine carries no
+  # GitHub SDK.
+  class GithubClient
+    API = "https://api.github.com"
+    TOKEN_URL = "https://github.com/login/oauth/access_token"
+    AUTHORIZE_URL = "https://github.com/login/oauth/authorize"
+    TIMEOUT_SECONDS = 8
+    # 100 per page is GitHub's maximum; five pages bounds a picker at 500.
+    PER_PAGE = 100
+    MAX_PAGES = 5
+
+    class Error < StandardError; end
+    # The token was revoked or expired: the owner has to connect again.
+    class Unauthorized < Error; end
+
+    def self.authorize_url(redirect_uri:, state:)
+      query = {
+        client_id: ActionAgent.github_client_id,
+        redirect_uri: redirect_uri,
+        scope: ActionAgent.github_oauth_scopes,
+        state: state,
+        allow_signup: "false"
+      }.to_query
+
+      "#{AUTHORIZE_URL}?#{query}"
+    end
+
+    # Trades an OAuth code for { access_token:, scope: }.
+    def self.exchange_code(code:, redirect_uri:)
+      uri = URI.parse(TOKEN_URL)
+      request = Net::HTTP::Post.new(uri, "Accept" => "application/json")
+      request.set_form_data(
+        client_id: ActionAgent.github_client_id,
+        client_secret: ActionAgent.github_client_secret,
+        code: code,
+        redirect_uri: redirect_uri
+      )
+      data = perform(uri, request)
+      raise Error, data["error_description"].presence || data["error"] if data["error"].present?
+      raise Error, "GitHub returned no access token" if data["access_token"].blank?
+
+      { access_token: data["access_token"], scope: data["scope"].to_s }
+    end
+
+    def self.perform(uri, request)
+      response = Net::HTTP.start(
+        uri.host, uri.port,
+        use_ssl: true, open_timeout: TIMEOUT_SECONDS, read_timeout: TIMEOUT_SECONDS
+      ) { |http| http.request(request) }
+
+      raise Unauthorized, "GitHub rejected the token" if response.code.to_i == 401
+      raise Error, "GitHub answered #{response.code}" unless response.code.to_i.between?(200, 299)
+
+      JSON.parse(response.body.presence || "{}")
+    rescue JSON::ParserError
+      raise Error, "GitHub answered with something other than JSON"
+    rescue Timeout::Error, SocketError, SystemCallError, OpenSSL::SSL::SSLError => e
+      raise Error, "GitHub is unreachable (#{e.class.name})"
+    end
+
+    def initialize(access_token)
+      @access_token = access_token
+    end
+
+    def user
+      get("/user")
+    end
+
+    # Every repository the token reaches (owned, collaborated on, or through
+    # an organization), most recently pushed first, reduced to the fields the
+    # dashboard stores.
+    def repositories
+      (1..MAX_PAGES).each_with_object([]) do |page, all|
+        batch = get("/user/repos", per_page: PER_PAGE, page: page, sort: "pushed",
+          affiliation: "owner,collaborator,organization_member")
+        all.concat(Array(batch).map { |repo| self.class.slice_repository(repo) })
+        break all if Array(batch).size < PER_PAGE
+      end
+    end
+
+    def self.slice_repository(repo)
+      {
+        "id" => repo["id"],
+        "full_name" => repo["full_name"],
+        "private" => repo["private"] == true,
+        "default_branch" => repo["default_branch"].presence || "main",
+        "description" => repo["description"],
+        "html_url" => repo["html_url"],
+        "pushed_at" => repo["pushed_at"]
+      }
+    end
+
+    private
+
+    def get(path, params = {})
+      uri = URI.parse("#{API}#{path}")
+      uri.query = params.to_query if params.any?
+      request = Net::HTTP::Get.new(uri,
+        "Accept" => "application/vnd.github+json",
+        "Authorization" => "Bearer #{@access_token}",
+        "X-GitHub-Api-Version" => "2022-11-28",
+        "User-Agent" => "activeagent-dashboard")
+      self.class.perform(uri, request)
+    end
+  end
+end