actionagent 1.8.0 → 1.9.0

data/app/models/action_agent/agent_template.rb

@@ -157,6 +157,40 @@ module ActionAgent
           icon: "🚀",
           featured: false
         },
+        {
+          name: "Conference Ticket Agent",
+          slug: "conference-ticket",
+          description: "Registers for a conference in the platform's browser and stops before paying: the run hands the ticket page to a person, who pays. Built for the SF Ruby Conference demo; works on any event page.",
+          category: "automation",
+          provider: "anthropic",
+          model: "claude-sonnet-5",
+          preset_type: "playwright",
+          appearance: { hat: "fedora", hatAccessory: "theaterMasks", heldItem: "browser" },
+          instruction_sets: [],
+          # playwright_mcp is the platform's browser (browser_navigate,
+          # browser_snapshot, browser_click) plus request_handoff.
+          tools: %w[playwright_mcp],
+          model_config: { temperature: 0.1, max_tokens: 4096 },
+          instructions: <<~INSTRUCTIONS.strip,
+            You register a person for a conference in a real browser, and you stop before anything is paid.
+
+            You are given the event URL, the attendee's name and email, and the ticket they want. The person who started this run is watching and pays themselves.
+
+            How to work:
+            1. browser_navigate to the event URL, then browser_snapshot to read the page.
+            2. Find the tickets or registration link (Tickets, Register, Get ticket, Buy). Ticket pages often live on a ticketing site such as Luma; follow the link.
+            3. Choose the ticket you were given. If it is sold out or missing, stop and say so.
+            4. Enter only the attendee details you were given: name, email, and anything else the person listed. Leave every other field empty.
+            5. Take a browser_snapshot after each step and use element refs from the latest snapshot only.
+            6. The moment the page asks for a card number, billing address, a wallet, a password, or a one-time code, call request_handoff with the current URL, what the page is asking for, and the values you entered. Then stop.
+
+            Never enter payment details. Never click Pay, Place order, Confirm purchase or Complete registration. Never accept terms on the person's behalf. Never invent details. Stay on the event's own pages and its ticketing site.
+
+            Report in three lines: where you stopped, what you entered, and what the person does next.
+          INSTRUCTIONS
+          icon: "🎟️",
+          featured: true
+        },
         {
           name: "PlaywrightMCP Demo",
           slug: "playwright-mcp-demo",
@@ -176,7 +210,7 @@ module ActionAgent
             {
               name: "playwright",
               command: "npx",
-              args: [ "-y", "@anthropic/mcp-server-playwright" ]
+              args: [ "-y", "@playwright/mcp@latest" ]
             }
           ],
           model_config: { temperature: 0.2, max_tokens: 4096 },

data/app/models/action_agent/api_key.rb

@@ -36,6 +36,20 @@ module ActionAgent
       update_column(:last_used_at, Time.current) if last_used_at.nil? || last_used_at < 1.minute.ago
     end
 
+    # The user who created the key, whom the MCP facade treats as the caller
+    # of a request made with it. Nil when the install has no user model, or
+    # the key records no creator: keys created before keys recorded one, and
+    # keys the engine creates for itself.
+    def creator
+      return nil if user_id.nil?
+
+      user_class = ActionAgent.user_class&.safe_constantize
+      return nil if user_class.nil?
+
+      reflection = self.class.reflect_on_association(:user)
+      reflection&.class_name == user_class.name ? user : user_class.find_by(id: user_id)
+    end
+
     # Safe to display in the dashboard key list: "aa_3xam…k9Q2"
     def masked_token
       "#{token_prefix}…#{token.last(4)}"

data/app/models/action_agent/catalog_scenario.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # One scenario of a catalog set, as the document gave it. It becomes an
+  # EvaluationScenario of the set's evaluation when the set is materialized;
+  # tags and params stay here, for the catalog's readers, since the
+  # evaluation's scenarios do not carry them.
+  class CatalogScenario < ApplicationRecord
+    belongs_to :set, class_name: "ActionAgent::ScenarioSet", foreign_key: :scenario_set_id, inverse_of: :scenarios
+
+    validates :key, presence: true, length: { maximum: 200 }, uniqueness: { scope: :scenario_set_id }
+    validates :prompt, presence: true
+
+    scope :enabled, -> { where(enabled: true) }
+
+    def expectations
+      value = self[:expectations]
+      value.is_a?(Hash) ? value : {}
+    end
+
+    def tags
+      Array(self[:tags]).map(&:to_s)
+    end
+
+    def params
+      value = self[:params]
+      value.is_a?(Hash) ? value : {}
+    end
+
+    # @return [ActiveAgent::Evals::Scenario]
+    def to_scenario
+      ActiveAgent::Evals::Scenario.from_hash(to_document.merge("position" => position), group: set.key, group_name: set.name)
+    end
+
+    def to_document
+      {
+        "key" => key,
+        "prompt" => prompt,
+        "expect" => expectations.presence,
+        "notes" => notes,
+        "tags" => tags.presence,
+        "params" => params.presence,
+        "production_only" => (true if production_only?)
+      }.compact
+    end
+
+    # The attributes Evaluation#replace_scenarios! takes.
+    def evaluation_attributes(index)
+      {
+        "key" => key,
+        "prompt" => prompt,
+        "group" => set.key,
+        "notes" => notes,
+        "expectations" => expectations,
+        "position" => position || index,
+        "enabled" => enabled
+      }
+    end
+
+    def summary
+      {
+        id: id,
+        key: key,
+        prompt: prompt,
+        notes: notes,
+        expectations: expectations,
+        tags: tags,
+        params: params,
+        production_only: production_only?,
+        enabled: enabled?,
+        position: position
+      }
+    end
+  end
+end

data/app/models/action_agent/code_session.rb

@@ -14,6 +14,9 @@ module ActionAgent
     owned_by :user, :account
 
     belongs_to :sandbox_session
+    belongs_to :evaluation_run, optional: true
+    belongs_to :verification_run, class_name: "EvaluationRun", optional: true
+    belongs_to :previous_code_session, class_name: "CodeSession", optional: true
 
     enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
 
@@ -22,8 +25,10 @@ module ActionAgent
     # Per string inside an event: tool results can be whole files.
     MAX_EVENT_STRING = 4_000
     MAX_DIFF_BYTES = 500_000
+    RUNNERS = %w[claude_code codex].freeze
 
     validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
+    validates :runner, inclusion: { in: RUNNERS }
 
     scope :recent, -> { order(created_at: :desc) }
 
@@ -47,11 +52,13 @@ module ActionAgent
       finished_at.nil?
     end
 
-    # The values that must never be stored: the checkout token and the
-    # Claude Code credential this session ran with.
+    # The values that must never be stored: the checkout token, the Claude
+    # Code credential this session ran with, and the secrets of the project
+    # the checkout was booted for. The session itself never receives those.
     def secrets
       spec = sandbox_session.checkout_spec rescue nil
-      [ spec&.dig(:token), *sandbox_session.runtime_environment.values ].compact
+      [ spec&.dig(:token), *sandbox_session.runtime_environment(runner: runner).values,
+        *sandbox_session.project_scrub_values ].compact
     end
 
     # Appends one stream-json event, scrubbed and bounded. Past MAX_EVENTS
@@ -77,7 +84,8 @@ module ActionAgent
 
       update!(
         result: event["result"].to_s.presence && SecretScrubber.scrub(event["result"].to_s, secrets).truncate(MAX_EVENT_STRING * 4),
-        claude_session_id: event["session_id"],
+        claude_session_id: runner == "claude_code" ? event["session_id"] : nil,
+        runner_session_id: event["session_id"],
         num_turns: event["num_turns"],
         duration_ms: event["duration_ms"],
         total_cost_usd: event["total_cost_usd"],
@@ -98,6 +106,13 @@ module ActionAgent
         status: status,
         prompt: prompt,
         model: model,
+        runner: runner,
+        credential_mode: credential_mode,
+        evaluation_run_id: evaluation_run_id,
+        evaluation_id: evaluation_run&.evaluation_id,
+        fix_item: fix_item,
+        verification_run_id: verification_run_id,
+        verification_error: verification_error,
         result: result,
         error_message: error_message,
         num_turns: num_turns,
@@ -123,7 +138,8 @@ module ActionAgent
         events: events.drop(after),
         events_offset: after,
         dropped_events_count: dropped_events_count.to_i,
-        diff: finished? ? diff : nil
+        diff: finished? ? diff : nil,
+        verification: evaluation_run_id ? EvaluationFixComparison.new(self).summary : nil
       )
     end
 

data/app/models/action_agent/draft_pull_request.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # A pull request opened from a checkout sandbox's changes (see
+  # DraftPullRequestPublisher), and how its last publish went.
+  #
+  # A publish is one of four operations:
+  #
+  #   create        a new branch and a draft pull request for it
+  #   update        a new commit on the branch of an opened pull request,
+  #                 with +commit_message+ as its message
+  #   open_draft    a draft pull request for a branch that was published
+  #                 without one (see #branch_only?)
+  #   open_regular  a regular pull request for such a branch, which only an
+  #                 explicit request opens
+  #
+  # +status+ is how the last one went: queued, publishing, published,
+  # draft_refused (the branch was pushed, the draft was not opened) or
+  # failed. A publish still queued or publishing STALL_AFTER after its last
+  # write is failed as stalled by .fail_stalled!. +state+ and +draft+ are
+  # what GitHub last reported about the pull request, read again at most
+  # once per STATUS_REFRESH_INTERVAL.
+  #
+  # The +user_id+ is the user who published, also where the owner is the
+  # account.
+  class DraftPullRequest < ApplicationRecord
+    include Ownable
+    owned_by :account, :user
+
+    OPERATIONS = %w[create update open_draft open_regular].freeze
+    STATUSES = %w[queued publishing published draft_refused failed].freeze
+    CREDENTIAL_KINDS = %w[app oauth].freeze
+    STATES = %w[open closed merged].freeze
+    STATUS_REFRESH_INTERVAL = 1.minute
+    STALL_AFTER = 15.minutes
+    DEFAULT_UPDATE_MESSAGE = "Update from the sandbox"
+    MAX_TITLE_CHARACTERS = 256
+    MAX_BODY_CHARACTERS = 20_000
+
+    belongs_to :sandbox_session
+
+    validates :repository, :branch, :base_commit, :title, presence: true
+    validates :title, length: { maximum: MAX_TITLE_CHARACTERS }
+    validates :body, :commit_message, length: { maximum: MAX_BODY_CHARACTERS }
+    validates :operation, inclusion: { in: OPERATIONS }
+    validates :status, inclusion: { in: STATUSES }
+    validates :credential_kind, inclusion: { in: CREDENTIAL_KINDS }, allow_nil: true
+    validates :state, inclusion: { in: STATES }, allow_nil: true
+
+    scope :recent, -> { order(created_at: :desc, id: :desc) }
+
+    # Fails every publish of the relation that is still queued or publishing
+    # STALL_AFTER after it was last written: its job died, or never ran.
+    #
+    # @return [Integer] how many were failed
+    def self.fail_stalled!(now: Time.current)
+      where(status: %w[queued publishing]).where(updated_at: ...(now - STALL_AFTER)).update_all(
+        status: "failed", error_code: "stalled", updated_at: now,
+        error_message: "The publish did not finish within #{STALL_AFTER.inspect}. Check the branch on GitHub, then publish again"
+      )
+    end
+
+    # JSON columns carry no default on MySQL or SQLite (see the migration).
+    def files
+      Array(super)
+    end
+
+    # Whether the last publish is still to run or running.
+    def in_progress?
+      status.in?(%w[queued publishing])
+    end
+
+    # Whether the pull request exists on GitHub.
+    def opened?
+      number.present?
+    end
+
+    # Whether the branch is on GitHub with no pull request for it: GitHub
+    # refused the draft, or opening it failed after the branch was pushed.
+    def branch_only?
+      head_commit.present? && !opened?
+    end
+
+    # The user who published, under the host's user model, or nil.
+    def publisher
+      user_class = ActionAgent.user_class&.safe_constantize
+      user_class && user_id ? user_class.find_by(id: user_id) : nil
+    end
+
+    # Claims the next status refresh for the caller when the last one is at
+    # least STATUS_REFRESH_INTERVAL old: true for exactly one of any callers
+    # racing for it.
+    def claim_status_refresh!(now: Time.current)
+      return false unless opened?
+
+      claimed = self.class.where(id: id)
+        .where("last_checked_at IS NULL OR last_checked_at < ?", now - STATUS_REFRESH_INTERVAL)
+        .update_all(last_checked_at: now)
+      self.last_checked_at = now if claimed == 1
+      claimed == 1
+    end
+
+    def summary
+      {
+        id: id,
+        sandbox_session_id: sandbox_session&.session_id,
+        repository: repository,
+        base_branch: base_branch,
+        branch: branch,
+        base_commit: base_commit,
+        head_commit: head_commit,
+        title: title,
+        body: body,
+        files: files.map { |file| file.slice("path", "status", "mode") },
+        operation: operation,
+        status: status,
+        error_code: error_code,
+        error_message: error_message,
+        credential_kind: credential_kind,
+        number: number,
+        url: url,
+        compare_url: compare_url,
+        state: state,
+        draft: draft,
+        last_checked_at: last_checked_at&.iso8601,
+        created_at: created_at&.iso8601,
+        updated_at: updated_at&.iso8601
+      }
+    end
+  end
+end

data/app/models/action_agent/evaluation.rb

@@ -11,6 +11,10 @@ module ActionAgent
   # agent's traces (error rate, latency); the llm_judge type asks a judge
   # model to score each sample and requires a configured provider.
   class Evaluation < ApplicationRecord
+    # Raised by #merge_scenarios! for a merge that would leave the suite
+    # holding more scenarios than its limit.
+    class ScenarioLimitExceeded < StandardError; end
+
     belongs_to :agent
     has_many :evaluation_runs, dependent: :destroy
     has_many :scenarios, class_name: "EvaluationScenario", dependent: :destroy
@@ -34,6 +38,20 @@ module ActionAgent
     validate :validate_criteria
 
     scope :recent, -> { order(updated_at: :desc) }
+    # Archived evaluations keep their runs but leave the index and its
+    # tiles unless asked for (see #archive!). The mark lives in the config
+    # JSON, so the filter reads it with each database's own JSON path.
+    scope :archived, -> { where("#{archived_at_sql} IS NOT NULL") }
+    scope :unarchived, -> { where("#{archived_at_sql} IS NULL") }
+
+    # SQL reading config["archived_at"] on the connected database.
+    def self.archived_at_sql
+      case connection.adapter_name.to_s.downcase
+      when /postgres/ then "#{quoted_table_name}.config ->> 'archived_at'"
+      when /mysql|trilogy/ then "JSON_UNQUOTE(JSON_EXTRACT(#{quoted_table_name}.config, '$.archived_at'))"
+      else "json_extract(#{quoted_table_name}.config, '$.archived_at')"
+      end
+    end
 
     # MySQL cannot give a JSON column a default, so a row inserted there
     # without `criteria` or `config` reads back nil. Both readers answer with
@@ -50,6 +68,56 @@ module ActionAgent
       evaluation_runs.order(created_at: :desc).first
     end
 
+    # The newest run that finished: the one the evaluation's pass rate
+    # describes. A newer run still pending, or one that failed, shows
+    # beside it and never in its place. Read from the loaded association
+    # when the index preloaded it.
+    def headline_run
+      if evaluation_runs.loaded?
+        evaluation_runs.select(&:complete?).max_by { |run| [ run.created_at, run.id ] }
+      else
+        evaluation_runs.complete.order(created_at: :desc, id: :desc).first
+      end
+    end
+
+    # --- archiving ---------------------------------------------------------
+    #
+    # An evaluation nobody maintains — its suite superseded, its agent
+    # retired — keeps its history but stops counting: the index leaves it
+    # out, and so do the pooled pass rate and the agent's scorecard. The
+    # mark is config["archived_at"]; a new run or a published report clears
+    # it, since either says the evaluation is alive after all.
+
+    def archived_at
+      value = config["archived_at"]
+      value.present? ? Time.zone.parse(value.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def archived?
+      config["archived_at"].present?
+    end
+
+    def archive!
+      update!(config: config.merge("archived_at" => Time.current.iso8601))
+    end
+
+    def unarchive!
+      return unless archived?
+
+      update!(config: config.except("archived_at"))
+    end
+
+    # Where this evaluation stands against the agent as it is now: whether
+    # its headline run scored the current version (EvaluationStanding).
+    # Memoized per instance; the index preloads what it needs.
+    def standing_info
+      @standing_info ||= EvaluationStanding.new(self)
+    end
+
+    attr_writer :standing_info
+
     def judge_defined?
       judge_kind == "judge_defined"
     end
@@ -127,12 +195,92 @@ module ActionAgent
       scenarios.reload
     end
 
+    # Adds the scenarios of +attributes+ (ActiveAgent::Evals::ScenarioParser
+    # output) whose keys the suite lacks, and updates the ones whose keys it
+    # holds. A scenario whose key +attributes+ do not name is never touched,
+    # unlike #replace_scenarios!, which treats its input as the whole suite.
+    #
+    #   - An existing key gets the given prompt, group, notes and
+    #     expectations, and a group, notes or expectations the attributes
+    #     leave out clears the stored one. It keeps its record, results,
+    #     position and enabled flag unless the attributes set "position" or
+    #     "enabled".
+    #   - A new key is appended after the suite's last position, in input
+    #     order, enabled unless the attributes say otherwise.
+    #
+    # ScenarioParser output carries each scenario's position within its own
+    # paste, which an existing key would take; remove "position" from parser
+    # output to keep the suite's order.
+    #
+    # Nothing is written when the merge raises.
+    #
+    # @param limit [Integer, nil] the most scenarios the suite may hold
+    #   afterwards
+    # @raise [ArgumentError] when a scenario has no key or two share one
+    # @raise [ScenarioLimitExceeded] when the suite would pass +limit+
+    # @return [Hash{Symbol => Array<String>}] the keys under :added, :updated
+    #   (an existing scenario that changed) and :unchanged
+    def merge_scenarios!(attributes, limit: nil)
+      keys = attributes.map { |attrs| attrs["key"].to_s }
+      raise ArgumentError, "every merged scenario needs a key" if keys.any?(&:blank?)
+
+      repeated = keys.tally.select { |_, count| count > 1 }.keys
+      raise ArgumentError, "scenario keys repeat: #{repeated.join(', ')}" if repeated.any?
+
+      merged = { added: [], updated: [], unchanged: [] }
+      # Locked, so two merges into one suite cannot both pass the limit or
+      # both add the same key.
+      with_lock do
+        existing = scenarios.where(key: keys).index_by(&:key)
+        added = keys.count { |key| !existing.key?(key) }
+        if limit && (held = scenarios.count) + added > limit
+          raise ScenarioLimitExceeded, "Scenario limit reached (#{limit} per evaluation): the evaluation holds #{held} " \
+                                       "and this merge adds #{added}"
+        end
+
+        next_position = (scenarios.maximum(:position) || -1) + 1
+        attributes.each do |attrs|
+          scenario = existing[attrs["key"].to_s]
+          if scenario.nil?
+            scenarios.create!(scenario_fields(attrs).merge(key: attrs["key"].to_s, position: next_position,
+                                                          enabled: attrs.fetch("enabled", true)))
+            next_position += 1
+            merged[:added] << attrs["key"].to_s
+            next
+          end
+
+          fields = scenario_fields(attrs)
+          # A row stored without expectations reads them as {}, and is not
+          # changed by being given {}.
+          fields.delete(:expectations) if scenario.expectations == fields[:expectations]
+          scenario.assign_attributes(fields)
+          scenario.position = attrs["position"] if attrs.key?("position")
+          scenario.enabled = attrs["enabled"] if attrs.key?("enabled")
+          if scenario.changed?
+            scenario.save!
+            merged[:updated] << scenario.key
+          else
+            merged[:unchanged] << scenario.key
+          end
+        end
+      end
+      merged
+    ensure
+      # create! adds each record to the association, and a rolled-back one
+      # stays there as a new record that the evaluation's next save writes.
+      scenarios.reset
+    end
+
     def llm_criteria
       criteria.select { |c| c["type"] == "llm_judge" }
     end
 
     private
 
+    def scenario_fields(attrs)
+      { prompt: attrs["prompt"], group: attrs["group"], notes: attrs["notes"], expectations: attrs["expectations"] || {} }
+    end
+
     def validate_criteria
       if criteria.blank?
         # judge_defined evaluations start empty — the judge authors the KPIs