actionagent 1.8.1 → 1.9.0

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -53,9 +53,12 @@ module ActionAgent
       #                            raised
       #   - judge_provider_error:  true when that lookup raised, as it does
       #                            when a stored key no longer decrypts
-      #   - model_providers:       the providers agent runs have credentials
-      #                            for, leaving out any whose credentials
-      #                            cannot be read
+      #   - model_providers:       the providers scenario replays have
+      #                            credentials for, leaving out any whose
+      #                            credentials cannot be read. Resolved for
+      #                            the caller replays run as, never the
+      #                            signed-in member, whose personal keys no
+      #                            replay uses.
       def index
         scope = evaluations_scope
         scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present?
@@ -69,7 +72,9 @@ module ActionAgent
           evaluations: evaluations.map { |evaluation| serialize(evaluation) },
           archived_count: archived_count,
           **judge_provider_fields(owner),
-          model_providers: AgentExecutionService.available_providers(owner)
+          model_providers: AgentExecutionService.available_providers(
+            owner, actor: ScenarioEvaluationRunner.replay_actor_for(owner)
+          )
         }
       end
 

data/app/controllers/action_agent/api/explorations_controller.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # Explorations and the review of their candidate scenarios: listing them,
+    # reading one with its candidates, submitting candidates found outside
+    # the dashboard, editing, rejecting and accepting candidates, and
+    # stopping a walk to review what it found. See Exploration.
+    #
+    # Everything is scoped to the caller's owner, and another owner's
+    # exploration reads as 404. Accepting writes into an evaluation, so it
+    # needs :replace_scenarios; storing and editing candidates changes only
+    # the exploration.
+    class ExplorationsController < BaseController
+      include EvaluationRunStarting
+
+      LIST_LIMIT = 50
+
+      before_action :require_owner!
+      before_action :set_exploration, except: [ :index, :create ]
+
+      rescue_from Exploration::InvalidCandidate, Exploration::CandidateLimitExceeded, with: :invalid_candidate
+      rescue_from Exploration::AcceptRefused, with: :accept_refused
+      rescue_from Evaluation::ScenarioLimitExceeded, with: :scenario_limit
+
+      # GET /api/explorations?project_id=&evaluation_id=
+      def index
+        scope = owned(Exploration).recent
+        scope = scope.where(project_id: params[:project_id].to_s) if params[:project_id].present?
+        scope = scope.where(evaluation_id: params[:evaluation_id].to_s) if params[:evaluation_id].present?
+
+        render json: { explorations: scope.limit(LIST_LIMIT).map(&:summary) }
+      end
+
+      # GET /api/explorations/:id
+      def show
+        render json: detail(@exploration)
+      end
+
+      # POST /api/explorations { project_id: | evaluation_id:, candidates: [...] }
+      # Stores candidates an agent outside the dashboard found, as a new
+      # exploration with source "external" that is ready for review. An
+      # observed agent's evaluation is refused, as the MCP facade's
+      # `explorations_submit` refuses it, unless a host adapter replays it.
+      def create
+        project, evaluation = submission_target
+        return if performed?
+
+        exploration = Exploration.build_for(project: project, evaluation: evaluation, user: current_user,
+          account: current_account, source: "external", status: "review", started_at: Time.current,
+          finished_at: Time.current)
+        exploration.save_with_candidates!(candidate_list)
+
+        render json: detail(exploration), status: :created
+      end
+
+      # PATCH /api/explorations/:id/candidates/:candidate_id
+      # { prompt:, group:, notes: (or rubric:), tools:, contains:, not_contains:, state: }
+      # An edit, a rejection (state: "rejected"), or reconsidering a
+      # rejected candidate (state: "proposed"); see
+      # Exploration#update_candidate!.
+      def update_candidate
+        candidate = @exploration.update_candidate!(params[:candidate_id], candidate_attributes(params))
+
+        render json: { candidate: candidate, exploration: @exploration.summary }
+      end
+
+      # POST /api/explorations/:id/accept { candidate_ids: [...], edits: { id => {...} } }
+      # Merges the candidates into the target evaluation (see
+      # Exploration#accept!). A refused candidate answers 422 with
+      # `problems` keyed by candidate id, and nothing is written.
+      #
+      # :replace_scenarios is asked about the evaluation the candidates
+      # merge into, or about the exploration when a project's accept will
+      # create that evaluation.
+      #
+      # With scenario_set_id, the candidates become scenarios of that set of
+      # one of the owner's scenario catalogs instead (see
+      # Exploration#accept_into_set!), and :replace_scenarios is asked about
+      # its catalog.
+      def accept
+        return accept_into_set if params[:scenario_set_id].present?
+        return unless authorize_action!(:replace_scenarios, @exploration.target_evaluation || @exploration)
+
+        merged = @exploration.accept!(params[:candidate_ids], edits: accept_edits)
+        evaluation = merged[:evaluation]
+
+        render json: detail(@exploration.reload).merge(
+          accepted: merged.slice(:added, :updated, :unchanged),
+          evaluation: evaluation_summary(evaluation)
+        )
+      end
+
+      # POST /api/explorations/:id/stop
+      # Ends a pending or running exploration and keeps what it found for
+      # review. 409 once it has stopped.
+      def stop
+        unless @exploration.stop!(reason: "stopped")
+          return render json: { error: "This exploration is not running", status: @exploration.status }, status: :conflict
+        end
+
+        render json: detail(@exploration.reload)
+      end
+
+      private
+
+      def accept_into_set
+        products = ScenarioProduct.where(scenario_catalog_id: owned(ScenarioCatalog).select(:id))
+        set = ScenarioSet.where(scenario_product_id: products.select(:id)).find(params[:scenario_set_id].to_s)
+        return unless authorize_action!(:replace_scenarios, set.catalog)
+
+        merged = @exploration.accept_into_set!(params[:candidate_ids], set: set, edits: accept_edits)
+        render json: detail(@exploration.reload).merge(
+          accepted: merged.slice(:added, :updated, :unchanged),
+          scenario_set: scenario_set_summary(set)
+        )
+      end
+
+      def scenario_set_summary(set)
+        catalog = set.catalog
+        { id: set.id, key: set.key, name: set.name, product_key: set.product.key,
+          catalog: { id: catalog.id, key: catalog.key, name: catalog.name } }
+      end
+
+      def set_exploration
+        @exploration = owned(Exploration).find(params[:id])
+      end
+
+      # [project, evaluation] the submission names, one of them nil, or nil
+      # after rendering why there is none.
+      def submission_target
+        if params[:project_id].present?
+          [ owned(Project).find(params[:project_id].to_s), nil ]
+        elsif params[:evaluation_id].present?
+          evaluation = Evaluation.joins(:agent).where(agent: owner_agents).find(params[:evaluation_id].to_s)
+          if unexecutable_scenario_run?(evaluation)
+            render json: { error: "Observed agents are read-only — duplicate this agent to create an executable copy" },
+              status: :unprocessable_entity
+            return nil
+          end
+
+          [ nil, evaluation ]
+        else
+          render json: { error: "Name the project_id or evaluation_id the candidates are for" }, status: :bad_request
+          nil
+        end
+      end
+
+      def candidate_list
+        list = params[:candidates]
+        list.is_a?(Array) ? list.map { |entry| entry.respond_to?(:to_unsafe_h) ? entry.to_unsafe_h : entry } : list
+      end
+
+      # The fields of a candidate edit present in +source+. Nested values
+      # arrive as Parameters and are read as plain data by the model.
+      def candidate_attributes(source)
+        keys = %w[prompt group notes rubric tools contains not_contains expectations state]
+        source = source.to_unsafe_h if source.respond_to?(:to_unsafe_h)
+        source.is_a?(Hash) ? source.stringify_keys.slice(*keys) : {}
+      end
+
+      def accept_edits
+        edits = params[:edits]
+        edits = edits.to_unsafe_h if edits.respond_to?(:to_unsafe_h)
+        return {} unless edits.is_a?(Hash)
+
+        edits.transform_values { |edit| candidate_attributes(edit).except("state") }
+      end
+
+      # The exploration with its candidates and what its review shows beside
+      # them: the evaluation they merge into, how many candidates to
+      # pre-select, and the owner's run allowance.
+      def detail(exploration)
+        evaluation = exploration.target_evaluation
+        usage = ActionAgent.usage_for(current_owner)
+        {
+          exploration: exploration.summary,
+          candidates: exploration.candidates,
+          project: exploration.project && { id: exploration.project.id, name: exploration.project.name },
+          target_agent: exploration.target_agent && { id: exploration.target_agent.id, name: exploration.target_agent.name },
+          evaluation: evaluation_summary(evaluation),
+          preselect_limit: ActionAgent.exploration_preselect_limit_for(current_owner),
+          runs_remaining: usage[:runs_remaining] || usage["runs_remaining"]
+        }
+      end
+
+      # What the review needs to estimate a run of the whole suite: the
+      # enabled scenarios, and the models each replays under.
+      def evaluation_summary(evaluation)
+        return nil if evaluation.nil?
+
+        {
+          id: evaluation.id,
+          name: evaluation.name,
+          enabled_scenario_count: evaluation.scenarios.enabled.count,
+          model_count: [ evaluation.compare_models.size, 1 ].max
+        }
+      end
+
+      def invalid_candidate(exception)
+        render json: { error: exception.message }, status: :unprocessable_entity
+      end
+
+      def accept_refused(exception)
+        render json: { error: exception.message, problems: exception.problems }, status: :unprocessable_entity
+      end
+
+      def scenario_limit(exception)
+        render json: { error: exception.message }, status: :unprocessable_entity
+      end
+    end
+  end
+end

data/app/controllers/action_agent/api/github_app_manifests_controller.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # Creating this dashboard's GitHub App from a manifest (Settings ->
+    # Integrations -> Create GitHub App), so a self-hoster need not register
+    # one by hand. Offered only when multi_tenant is off: a multi-tenant
+    # platform registers its App once, per environment, outside the
+    # dashboard.
+    #
+    # The dashboard posts the manifest #create returns to GitHub, which
+    # creates the App and returns to #callback with a code. #callback
+    # converts it and shows the App's settings and credentials on a page of
+    # their own, once. Nothing here stores them: the operator adds them to
+    # the dashboard's configuration (see ActionAgent.github_app_id).
+    class GithubAppManifestsController < BaseController
+      STATE_SESSION_KEY = :action_agent_github_app_manifest_state
+      # The permissions the App requests. Contents and pull requests are
+      # written only by an explicit publish; the organization members read
+      # is what an installation's admin check needs. No workflows,
+      # administration or secrets permission.
+      PERMISSIONS = { contents: "write", pull_requests: "write", metadata: "read", members: "read" }.freeze
+      # GitHub's limit on an App's name.
+      MAX_NAME_LENGTH = 34
+
+      before_action :require_owner!
+      before_action :require_single_tenant!
+      before_action :authorize_github!
+
+      # POST /api/github_app_manifest { organization: "acme" (optional) }
+      #
+      # Returns the GitHub URL to post the manifest to, with a single-use
+      # state, and the manifest itself, which the dashboard submits as the
+      # form field +manifest+.
+      def create
+        organization = params[:organization].presence
+        unless organization.nil? || (organization.is_a?(String) && organization.match?(GithubClient::LOGIN))
+          return render json: { error: "organization must be a GitHub organization login" }, status: :bad_request
+        end
+
+        state = SecureRandom.urlsafe_base64(32)
+        session[STATE_SESSION_KEY] = { "state" => state, "user_id" => current_user_id }.compact
+
+        render json: { url: GithubClient.new_app_url(state: state, organization: organization), manifest: manifest }
+      end
+
+      # GET /api/github_app_manifest/callback?code=...&state=...
+      def callback
+        # Single use: read and cleared before anything else can fail.
+        issued = session.delete(STATE_SESSION_KEY)
+        return redirect_to_settings(github_app: "invalid_state") unless issued_here?(issued, params[:state])
+        return redirect_to_settings(github_app: "missing_code") unless params[:code].is_a?(String) && params[:code].present?
+
+        @app = GithubClient.convert_manifest(params[:code])
+        @settings_url = "#{request.script_name}/settings?tab=integrations"
+
+        # The page carries the App's secrets: no cache keeps it, no other
+        # origin's script runs on it, and its links send no referrer.
+        response.headers["Cache-Control"] = "no-store"
+        response.headers["Referrer-Policy"] = "no-referrer"
+        response.headers["Content-Security-Policy"] =
+          "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
+        render template: "action_agent/github_app_manifests/created", layout: false
+      rescue GithubClient::Error => e
+        Rails.logger.warn("[ActionAgent] GitHub App manifest conversion failed: #{e.message}")
+        redirect_to_settings(github_app: "manifest_error")
+      end
+
+      private
+
+      def manifest
+        base = "#{request.base_url}#{request.script_name}"
+
+        {
+          name: "ActiveAgent #{request.host}".first(MAX_NAME_LENGTH),
+          url: base.presence || request.base_url,
+          redirect_url: "#{base}/api/github_app_manifest/callback",
+          callback_urls: [ "#{base}/api/github_installations/callback" ],
+          request_oauth_on_install: true,
+          # Asks GitHub to return after an existing installation is
+          # reconfigured too, which is how one is linked again.
+          setup_on_update: true,
+          public: false,
+          default_permissions: PERMISSIONS,
+          default_events: []
+        }
+      end
+
+      def issued_here?(issued, given)
+        issued.is_a?(Hash) && issued["state"].is_a?(String) && given.is_a?(String) &&
+          ActiveSupport::SecurityUtils.secure_compare(issued["state"], given) &&
+          issued["user_id"] == current_user_id
+      end
+
+      def current_user_id
+        current_user.respond_to?(:id) ? current_user.id.to_s : nil
+      end
+
+      def require_single_tenant!
+        return unless ActionAgent.multi_tenant?
+        return redirect_to_settings(github_app: "manifest_unavailable") if action_name == "callback"
+
+        render json: { error: "Creating a GitHub App is not available on a multi-tenant dashboard", code: "manifest_unavailable" },
+          status: :not_found
+      end
+
+      def authorize_github!
+        authorize_action!(:manage_github, owned(GithubInstallation).new)
+      end
+
+      # The callback is a browser navigation, so a refusal returns to
+      # Settings, discarding the state.
+      def permission_denied(action)
+        return super unless action_name == "callback"
+
+        session.delete(STATE_SESSION_KEY)
+        redirect_to_settings(github_app: "forbidden")
+      end
+
+      def redirect_to_settings(**query)
+        redirect_to "#{request.script_name}/settings?#{{ tab: 'integrations' }.merge(query).to_query}"
+      end
+    end
+  end
+end

data/app/controllers/action_agent/api/github_connections_controller.rb

@@ -15,19 +15,23 @@ module ActionAgent
       before_action :require_owner!
       before_action :require_github_oauth!, only: [ :connect, :callback ]
       before_action :set_connection, only: [ :repositories, :update, :destroy ]
+      before_action :authorize_github!, only: [ :connect, :callback, :update, :destroy ]
 
       # Later handlers win, so the subclass is registered last.
       rescue_from GithubClient::Error, with: :github_unavailable
       rescue_from GithubClient::Unauthorized, with: :github_unauthorized
 
-      # GET /api/github_connection
+      # GET /api/github_connection — the OAuth connection, and the GitHub App
+      # path beside it under +app+: whether an App is configured, whether one
+      # can be created from a manifest here, and the owner's installations.
       def show
         connection = owned(GithubConnection).first
 
         render json: {
           configured: ActionAgent.github_oauth_configured?,
           connected: connection.present?,
-          connection: connection&.as_summary
+          connection: connection&.as_summary,
+          app: github_app_status
         }
       end
 
@@ -100,6 +104,9 @@ module ActionAgent
         # A different GitHub account starts with nothing selected: the old
         # selection was checked against the other account's access.
         connection.repositories = [] if connection.persisted? && connection.github_user_id != user["id"]
+        # The user who authorized, also where the account owns the
+        # connection: only they may publish pull requests with its token.
+        connection.user_id = current_user.id if ActionAgent.user_class.present? && current_user.respond_to?(:id)
         connection.update!(
           access_token: grant[:access_token],
           scopes: grant[:scope],
@@ -120,6 +127,23 @@ module ActionAgent
         @connection = owned(GithubConnection).first!
       end
 
+      # Asks about the owner's connection, or about an unsaved one when
+      # #connect or #callback is about to create it.
+      def authorize_github!
+        subject = @connection || owned(GithubConnection).first || owned(GithubConnection).new
+        authorize_action!(:manage_github, subject)
+      end
+
+      # #connect and #callback are browser navigations, so a refusal returns
+      # to Settings like their other outcomes. The callback's state is
+      # discarded with it.
+      def permission_denied(action)
+        return super unless action_name.in?(%w[connect callback])
+
+        session.delete(STATE_SESSION_KEY)
+        redirect_to_settings(github: "forbidden")
+      end
+
       def require_github_oauth!
         return if ActionAgent.github_oauth_configured?
 
@@ -130,6 +154,20 @@ module ActionAgent
         "#{request.base_url}#{request.script_name}/api/github_connection/callback"
       end
 
+      def github_app_status
+        configured = ActionAgent.github_app_configured?
+
+        {
+          configured: configured,
+          slug: configured ? ActionAgent.github_app_slug : nil,
+          callback_url: "#{request.base_url}#{request.script_name}/api/github_installations/callback",
+          manifest_available: !ActionAgent.multi_tenant?,
+          # Listed only while an App is configured: without one nothing can
+          # mint an installation's tokens, so its repositories cannot be used.
+          installations: configured ? owned(GithubInstallation).order(:id).map(&:as_summary) : []
+        }
+      end
+
       def redirect_to_settings(**query)
         redirect_to "#{request.script_name}/settings?#{{ tab: 'integrations' }.merge(query).to_query}"
       end