actionagent 1.9.4 → 1.9.5

data/app/services/action_agent/evaluation_report_import.rb

@@ -17,9 +17,18 @@ module ActionAgent
   #   evaluation — that agent's evaluation for the `suite` and the report's
   #                scope (SCOPE_KEYS in its metadata), named for both
   #   scenarios  — one per reported scenario key, updated to the reported prompt
-  #   run        — one complete EvaluationRun per tenant and run_id, with one
+  #   run        — one EvaluationRun per tenant and run_id, with one
   #                EvaluationScenarioResult per scenario and model
   #
+  # An envelope's `status` is "complete" when absent. A "running" envelope
+  # stores the results so far as a running run, and each later delivery under
+  # its run_id replaces them, until a "complete" one finishes the run:
+  #   running  — "running" or "complete" replaces the stored results
+  #   complete — "complete" is an identical retry or a Conflict, as for any
+  #              stored run; "running" changes nothing, so a progress
+  #              delivery that lands after the final one is harmless
+  # Only the first delivery under a run_id is admitted and counted.
+  #
   # The run's per-model summary, criterion scores and recommendations are
   # computed from the stored results, not taken from the report. The judge's
   # verdict and label are taken from it, and so is what the judge spent —
@@ -85,6 +94,11 @@ module ActionAgent
     MAX_JUDGE_KINDS = 10
     SCOPE_PATTERN = %r{\A[\w .:/@-]{1,100}\z}
     STATUSES = %w[passed failed errored].freeze
+    # The envelope statuses a delivery may name.
+    RUN_STATUSES = %w[running complete].freeze
+    # What this collector accepts beyond a version-1 report, sent with its
+    # refusal of a payload that is not one so a publisher can ask before a run.
+    CAPABILITIES = %w[progress].freeze
     # Report metadata that tells one evaluation of a suite from another, in the
     # order it is written into the evaluation's name.
     SCOPE_KEYS = %w[scope environment role].freeze
@@ -102,20 +116,22 @@ module ActionAgent
     # The evaluation_runs columns an import writes that later migrations added.
     REQUIRED_RUN_COLUMNS = %w[external_tenant external_run_id external_report_digest agent_version_id].freeze
 
-    # Returns `[run, duplicate]`: the stored EvaluationRun, and whether an
-    # earlier identical delivery had already stored it. `account` is the
-    # tenant, or nil on a single-tenant install.
+    # Returns `[run, duplicate, progressed]`: the stored EvaluationRun, whether
+    # an earlier delivery had already stored this report, and whether this
+    # delivery replaced the results of a running run. `account` is the tenant,
+    # or nil on a single-tenant install.
     #
-    # `admit` is asked only for a report that would be stored, never for an
-    # identical retry. It returns nil to allow the report, or a denial, which
-    # is raised as Denied.
+    # `admit` is asked only for a run_id the tenant holds no run under. It
+    # returns nil to allow the report, or a denial, which is raised as Denied.
     #
     # The envelope's identities are validated as sent. NUL characters, which
     # PostgreSQL cannot store, are removed from every string inside `report`.
     #
     # Raises Invalid for a payload that is not a valid version-1 report, or an
     # evaluation name the agent already holds for another suite or scope;
-    # Conflict for different content under a stored run_id; Refused (and
+    # Conflict for different content under a complete run_id, or a delivery
+    # under a running run_id that the dashboard executed or that names another
+    # agent, suite or scope; Refused (and
     # LimitExceeded) when storing needs an operator to act first; Denied; and
     # Busy.
     def self.call(payload:, account: nil, admit: nil)
@@ -162,24 +178,25 @@ module ActionAgent
       validate!
       digest = self.class.digest(@payload)
       existing = stored_run
-      return [ identical!(existing, digest), true ] if existing
+      return settled(existing, digest) if settles?(existing, digest)
 
       attempts = 0
       begin
         OwnerLock.new(EvaluationRun.connection, owner_lock_key).synchronize do
           EvaluationRun.transaction(requires_new: true) do
             existing = stored_run
-            next [ identical!(existing, digest), true ] if existing
+            next settled(existing, digest) if settles?(existing, digest)
+            next [ progress(existing, digest), false, true ] if existing
 
             admit!
-            [ import(digest), false ]
+            [ import(digest), false, false ]
           end
         end
       rescue ActiveRecord::RecordNotUnique
         # A concurrent delivery committed this run, or an agent, evaluation or
         # scenario this import was creating, first. Reading again sees it.
         existing = stored_run
-        return [ identical!(existing, digest), true ] if existing
+        return settled(existing, digest) if settles?(existing, digest)
 
         attempts += 1
         retry if attempts < 2
@@ -265,6 +282,29 @@ module ActionAgent
       EvaluationReportExport.find(@payload["run_id"], account: @account)
     end
 
+    def running_delivery?
+      @payload["status"] == "running"
+    end
+
+    # Whether a delivery under a stored run's run_id leaves its results as
+    # they are. Only a running run this collector imported takes new ones.
+    def settles?(run, digest)
+      return false unless run
+      return true unless run.running? && run.imported?
+
+      run.external_report_digest == digest
+    end
+
+    # Returns `[run, true, false]` for a delivery that leaves the stored run as
+    # it is. Raises Conflict for a run the dashboard is executing, and for a
+    # different "complete" report under a run that has finished.
+    def settled(run, digest)
+      raise Conflict, "run_id names a run the dashboard is executing" if run.running? && !run.imported?
+      return [ run, true, false ] if running_delivery? || run.running?
+
+      [ identical!(run, digest), true, false ]
+    end
+
     # The stored run, when the payload is the report it was stored from or
     # the run's own export (EvaluationReportExport), which is how an
     # exported run posted back to its install is told from a different
@@ -286,26 +326,60 @@ module ActionAgent
       raise Denied, denial if denial.present?
     end
 
+    # Replaces a running run's results with the delivery's.
+    def progress(run, digest)
+      agent = find_or_create_agent
+      evaluation = find_or_initialize_evaluation(agent)
+      raise Conflict, "run_id already exists for another agent, suite or scope" unless evaluation.id == run.evaluation_id
+
+      scenarios = upsert_scenarios(evaluation)
+      run.scenario_results.destroy_all
+      run.update!(run_attributes(agent, digest))
+      store_results(run, scenarios, agent)
+    end
+
+    # A new run is dated by the time its report says it finished, when it
+    # names one (see #finished_at), so a run exported from one install and
+    # imported into another keeps its place in the run history.
     def import(digest)
       agent = find_or_create_agent
       evaluation = find_or_initialize_evaluation(agent)
       scenarios = upsert_scenarios(evaluation)
       run = evaluation.evaluation_runs.create!(
-        external_tenant: tenant_key,
-        external_run_id: @payload["run_id"],
+        run_attributes(agent, digest).merge(external_tenant: tenant_key, external_run_id: @payload["run_id"], created_at: finished_at)
+      )
+      store_results(run, scenarios, agent)
+    end
+
+    def run_attributes(agent, digest)
+      {
         external_report_digest: digest,
         agent_version: reported_release_version(agent),
-        status: :complete,
+        status: running_delivery? ? :running : :complete,
         selection: selection,
         scores: recorded_scores,
         samples_evaluated: results.size,
         samples_passed: results.count { |result| result["status"] == "passed" },
-        completed_at: Time.current
-      )
+        completed_at: running_delivery? ? nil : finished_at
+      }
+    end
+
+    # The report's `metadata.finished`, which EvaluationRun#to_report records
+    # as the run's completion time, or now when the report names none. A
+    # time later than now is read as now.
+    def finished_at
+      @finished_at ||= begin
+        now = Time.current
+        reported = metadata["finished"] && Time.iso8601(metadata["finished"])
+        reported && reported < now ? reported : now
+      end
+    end
+
+    def store_results(run, scenarios, agent)
       results.each { |result| persist(run, scenarios.fetch(result["scenario_key"]), result) }
       run.update!(scores: summarized_scores(run))
-      evaluation.touch
-      agent.update_columns(last_observed_at: run.completed_at, updated_at: Time.current)
+      run.evaluation.touch
+      agent.update_columns(last_observed_at: Time.current, updated_at: Time.current)
       run
     end
 
@@ -621,6 +695,9 @@ module ActionAgent
       object!(@payload, "payload")
       validate_json!(@payload)
       raise Invalid, "version must be 1" unless @payload["version"] == 1
+      unless @payload["status"].nil? || RUN_STATUSES.include?(@payload["status"])
+        raise Invalid, "status must be one of #{RUN_STATUSES.join(', ')}"
+      end
 
       %w[run_id source suite].each { |key| identifier!(@payload[key], key) }
       unless @payload["agent_name"].is_a?(String) && AGENT_NAME_PATTERN.match?(@payload["agent_name"]) && @payload["agent_name"].strip.length >= 2
@@ -634,6 +711,7 @@ module ActionAgent
         raise Invalid, "suite and scope name an evaluation longer than #{MAX_EVALUATION_NAME} characters"
       end
 
+      validate_finished!
       judge_trace_ids!(metadata["judge_trace_ids"])
       string!(report["judge"], "report.judge", 200)
       validate_release!
@@ -736,6 +814,21 @@ module ActionAgent
       string!(judge.dig("suggested_tool", "name"), "result.diagnosis.judge.suggested_tool.name", 200)
     end
 
+    def validate_finished!
+      finished = metadata["finished"]
+      return if finished.nil?
+      return if finished.is_a?(String) && finished.length <= 40 && iso8601?(finished)
+
+      raise Invalid, "report.metadata.finished must be an ISO 8601 time"
+    end
+
+    def iso8601?(value)
+      Time.iso8601(value)
+      true
+    rescue ArgumentError
+      false
+    end
+
     # The release the report says it evaluated: a digest the dashboard can
     # match to a version cut on deploy, and the deploy and label as text.
     def validate_release!

data/app/services/action_agent/github_client.rb

@@ -60,6 +60,10 @@ module ActionAgent
     # cannot see, which GitHub does not tell apart. Also 409, which GitHub
     # answers for the commits and trees of a repository with no commits.
     class NotFound < Error; end
+    # GitHub answered 403 to an organization membership lookup: the GitHub
+    # App lacks the organization permission Members (read), or the
+    # organization's SAML single sign-on has not authorized the token.
+    class MembershipForbidden < Error; end
     # GitHub minted no token because the installation was removed from its
     # account (404) or suspended (403). +reason+ is :removed or :suspended.
     class InstallationUnavailable < Error
@@ -323,14 +327,19 @@ module ActionAgent
 
     # The user's membership in +organization+ ({ "state" =>, "role" => }), or
     # nil when GitHub reports none.
+    #
+    # @raise [MembershipForbidden] when GitHub answers 403, as it does to a
+    #   GitHub App user token while the App lacks Members (read)
     def organization_membership(organization)
       raise Error, "#{organization.inspect} is not a GitHub login" unless organization.is_a?(String) && organization.match?(LOGIN)
 
       get("/user/memberships/orgs/#{organization}")
+    rescue NotFound
+      nil
     rescue Error => e
-      raise unless e.status.in?([ 403, 404 ])
+      raise unless e.status == 403
 
-      nil
+      raise MembershipForbidden.new("GitHub refused to show the membership in #{organization}: #{e.message}", status: 403)
     end
 
     # The repositories an installation token reaches, reduced like

data/app/services/action_agent/local_sandbox_claude_login.rb

@@ -108,7 +108,7 @@ module ActionAgent
       data = JSON.parse(File.read(path, 8192))
       status = data["status"].to_s
       result = { status: %w[starting awaiting_code submitted completed failed expired cancelled].include?(status) ? status : "failed" }
-      if status == "awaiting_code" && data["authorize_url"].to_s.match?(%r{\Ahttps://(?:claude\.ai|platform\.claude\.com)/oauth/authorize\?[^\s]+\z})
+      if status == "awaiting_code" && data["authorize_url"].to_s.match?(%r{\Ahttps://(?:claude\.ai|claude\.com|platform\.claude\.com)/(?:cai/)?oauth/authorize\?[^\s]+\z})
         result[:authorize_url] = data["authorize_url"]
       end
       result

data/app/services/action_agent/project_capabilities.rb

@@ -14,7 +14,8 @@ module ActionAgent
   #                      project's secrets and bootstrap reach it
   #   execution          agent execution is enabled
   #   github             a GitHub App or OAuth app is configured
-  #   github_connection  the owner connected GitHub
+  #   github_connection  the owner linked a usable GitHub App installation
+  #                      or connected GitHub through OAuth
   #   model_credentials  credentials for the model the project's agent
   #                      runs on (not blocking)
   #   browser            a browser for the project's sandbox, started and
@@ -24,7 +25,8 @@ module ActionAgent
     Item = Struct.new(:key, :label, :ok, :blocking, :detail, :fix, keyword_init: true)
 
     # @param owner [Object, nil] the caller's owner, for its credentials
-    # @param github_connected [Boolean]
+    # @param github_connected [Boolean] whether the owner has a usable GitHub
+    #   App installation or an OAuth connection
     # @param base_url [String] the engine's absolute mount URL, for callback URLs
     # @param environment [ActiveSupport::EnvironmentInquirer]
     def initialize(owner:, github_connected:, base_url:, environment: Rails.env)
@@ -137,9 +139,10 @@ module ActionAgent
     end
 
     def github_connection_item
+      fix = github[:mode] == "app" ? "Install the GitHub App in Settings → Integrations" : "Connect GitHub in Settings → Integrations"
       Item.new(key: "github_connection", label: "GitHub connection", ok: @github_connected, blocking: true,
         detail: @github_connected ? "Connected" : "GitHub is not connected",
-        fix: @github_connected ? nil : "Connect GitHub in Settings → Integrations")
+        fix: @github_connected ? nil : fix)
     end
 
     def default_model

data/config/routes.rb

@@ -134,6 +134,7 @@ ActionAgent::Engine.routes.draw do
     resources :projects, only: [ :index, :show, :create, :update, :destroy ] do
       collection do
         get :capabilities
+        get :repositories
         get :preflight
         get :discover_secrets
       end
@@ -318,6 +319,11 @@ ActionAgent::Engine.routes.draw do
       end
     end
 
+    # The GitHub App's webhook deliveries. Authenticated by GitHub's
+    # signature (github_app_webhook_secret), not a session; the App's
+    # Webhook URL is <mount>/api/github_app/webhooks.
+    post "github_app/webhooks", to: "github_app_webhooks#create", as: :github_app_webhooks
+
     # Creating the GitHub App from a manifest on a self-hosted dashboard.
     resource :github_app_manifest, only: [ :create ] do
       get :callback
@@ -365,6 +371,15 @@ ActionAgent::Engine.routes.draw do
   # data. See SessionPlayerController.
   get "session_player", to: "session_player#show", as: :session_player
 
+  # Shareable evaluation links: an evaluation, one of its runs, or one
+  # scenario result of a run. The React app reads these paths like any other
+  # client-side route; they are matched here first so the page itself is
+  # checked against the caller's scope, and a link to an evaluation outside
+  # it answers 404 whether or not the evaluation exists.
+  get "evaluations/:evaluation_id(/runs/:run_id(/results/:result_id))", to: "dashboard#evaluation",
+    as: :evaluation_link, format: false,
+    constraints: { evaluation_id: /\d+/, run_id: /\d+/, result_id: /\d+/ }
+
   # Everything else under the mount is a client-side route: render the
   # dashboard and let the browser resolve it. Anchored last so it can only
   # ever catch what the routes above did not, and refuses /api paths so a

data/lib/action_agent/claude_login_process.rb

@@ -76,7 +76,7 @@ begin
       buffer << chunk.force_encoding("UTF-8").scrub
       buffer = buffer[-32_768..] || buffer
       clean = buffer.gsub(/\e\[[0-9;?]*[A-Za-z]/, "")
-      url = clean[%r{https://(?:claude\.ai|platform\.claude\.com)/oauth/authorize\?[^\s\e]+}]
+      url = clean[%r{https://(?:claude\.ai|claude\.com|platform\.claude\.com)/(?:cai/)?oauth/authorize\?[^\s\e\a]+}]
       if url && URI.parse(url).query.to_s.include?("client_id=")
         write_status.call("awaiting_code", url)
       end

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.9.4"
+  VERSION = "1.9.5"
 end

data/lib/action_agent.rb

@@ -867,6 +867,16 @@ module ActionAgent
     attr_writer :github_app_id, :github_app_private_key, :github_app_slug,
       :github_app_client_id, :github_app_client_secret
 
+    # The secret GitHub signs the App's webhook deliveries with, which
+    # POST <mount>/api/github_app/webhooks verifies. Falls back to
+    # GITHUB_APP_WEBHOOK_SECRET. Not one of the settings
+    # github_app_configured? needs: without it the endpoint answers 404 and
+    # the dashboard learns of installation changes only when a token mint is
+    # refused. Set the App's Webhook URL to <mount>/api/github_app/webhooks
+    # and subscribe it to Installation and Installation repositories.
+    # @return [String, nil]
+    attr_writer :github_app_webhook_secret
+
     # MCP servers the host app itself serves or connects, appended to the
     # built-in catalog (MCPCatalog) so the MCP Services view lists them and
     # telemetry traffic attributes to them. Each entry is a hash shaped like
@@ -971,6 +981,10 @@ module ActionAgent
       @github_app_client_secret.presence || ENV["GITHUB_APP_CLIENT_SECRET"].presence
     end
 
+    def github_app_webhook_secret
+      @github_app_webhook_secret.presence || ENV["GITHUB_APP_WEBHOOK_SECRET"].presence
+    end
+
     # Whether the GitHub App installation flow can run on this install: every
     # github_app_* setting is present.
     # @return [Boolean]
@@ -1334,6 +1348,7 @@ module ActionAgent
       @github_app_slug = nil
       @github_app_client_id = nil
       @github_app_client_secret = nil
+      @github_app_webhook_secret = nil
       @trace_retention = nil
       @trace_owner_resolver = nil
       @usage_recorder = nil

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -215,6 +215,12 @@ ActionAgent.configure do |config|
   # config.github_app_client_secret = Rails.application.credentials.dig(:github_app, :client_secret)
   # config.github_app_private_key = Rails.application.credentials.dig(:github_app, :private_key)
   #
+  # With the App's webhook pointed at <mount>/api/github_app/webhooks, the
+  # secret its deliveries are signed with (GITHUB_APP_WEBHOOK_SECRET when
+  # unset). Settings then learns of an uninstall or suspension when it happens.
+  #
+  # config.github_app_webhook_secret = Rails.application.credentials.dig(:github_app, :webhook_secret)
+  #
   # Or an OAuth App, connected per owner with the callback URL
   # <mount>/api/github_connection/callback (GITHUB_CLIENT_ID and
   # GITHUB_CLIENT_SECRET when unset).

data/lib/generators/action_agent/templates/migrations/029_create_active_agent_github_webhook_deliveries.rb.erb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# The GitHub App webhook deliveries the dashboard accepted, kept briefly so a
+# redelivery of the same X-GitHub-Delivery is a no-op (see
+# ActionAgent::GithubWebhookDelivery). Nothing here is a credential or a
+# payload.
+class CreateActiveAgentGithubWebhookDeliveries < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}github_webhook_deliveries" do |t|
+      # GitHub's X-GitHub-Delivery, a GUID. Unique: a second insert of the
+      # same delivery is how a redelivery is recognized.
+      t.string :delivery_id, null: false, limit: 64
+      t.string :event, null: false, limit: 64
+      t.string :event_action, limit: 64
+      t.bigint :installation_id
+      t.datetime :created_at, null: false
+      t.index :delivery_id, unique: true, name: "index_#{prefix}webhook_deliveries_on_delivery"
+      t.index :created_at, name: "index_#{prefix}webhook_deliveries_on_created_at"
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.9.4
+  version: 1.9.5
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-25 00:00:00.000000000 Z
+date: 2026-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -142,6 +142,7 @@ files:
 - app/controllers/action_agent/api/feedback_ingest_controller.rb
 - app/controllers/action_agent/api/feedbacks_controller.rb
 - app/controllers/action_agent/api/github_app_manifests_controller.rb
+- app/controllers/action_agent/api/github_app_webhooks_controller.rb
 - app/controllers/action_agent/api/github_connections_controller.rb
 - app/controllers/action_agent/api/github_installations_controller.rb
 - app/controllers/action_agent/api/input_requests_controller.rb
@@ -194,6 +195,7 @@ files:
 - app/jobs/action_agent/evaluation_replay_job.rb
 - app/jobs/action_agent/evaluation_run_job.rb
 - app/jobs/action_agent/exploration_job.rb
+- app/jobs/action_agent/github_webhook_job.rb
 - app/jobs/action_agent/input_request_expiry_job.rb
 - app/jobs/action_agent/process_telemetry_traces_job.rb
 - app/jobs/action_agent/project_evaluation_job.rb
@@ -224,6 +226,7 @@ files:
 - app/models/action_agent/exploration.rb
 - app/models/action_agent/github_connection.rb
 - app/models/action_agent/github_installation.rb
+- app/models/action_agent/github_webhook_delivery.rb
 - app/models/action_agent/input_request.rb
 - app/models/action_agent/interaction_feedback.rb
 - app/models/action_agent/model_pricing.rb
@@ -248,6 +251,7 @@ files:
 - app/queries/action_agent/metrics_report.rb
 - app/queries/action_agent/trace_window_report.rb
 - app/serializers/action_agent/agent_message_serializer.rb
+- app/serializers/action_agent/agent_version_serializer.rb
 - app/serializers/action_agent/code_conversation_serializer.rb
 - app/serializers/action_agent/evaluation_serializer.rb
 - app/serializers/action_agent/input_request_serializer.rb
@@ -372,6 +376,7 @@ files:
 - lib/generators/action_agent/templates/migrations/026_add_evaluation_fixes_to_code_sessions.rb.erb
 - lib/generators/action_agent/templates/migrations/027_create_active_agent_interaction_feedbacks.rb.erb
 - lib/generators/action_agent/templates/migrations/028_add_conversations_to_code_sessions.rb.erb
+- lib/generators/action_agent/templates/migrations/029_create_active_agent_github_webhook_deliveries.rb.erb
 - lib/tasks/action_agent.rake
 - lib/tasks/browser.rake
 - lib/tasks/claude_code.rake