actionagent 1.9.7 → 1.9.8

data/app/models/action_agent/agent_run.rb

@@ -313,18 +313,37 @@ module ActionAgent
       LiveUpdates.broadcast("agent_runs_#{agent_id}", type: "update", id: id, status: status)
     end
 
+    # Statuses a cancel moves to `cancelled`. A finished run keeps its end.
+    CANCELLABLE_STATUSES = %i[pending running awaiting_input].freeze
+
     # Cancels a run that is in progress or waiting for input, and every
-    # request for input it still has pending.
+    # request for input it still has pending. Returns whether this call
+    # cancelled it.
+    #
+    # The status is checked by the write itself, as AgentExecutionJob claims
+    # a pending run, because this copy of the run may be older than the
+    # database's: the request's, or a caller's that has held it since it
+    # executed. A run that finished in the meantime is left as it finished,
+    # and this copy is reloaded to show how. The update skips callbacks, so
+    # the cancel is broadcast here, once.
     #
     # @param reason [String] recorded as the run's error message
+    # @return [Boolean]
     def cancel!(reason = "Cancelled by user")
-      return unless in_progress? || awaiting_input?
+      return false unless in_progress? || awaiting_input?
 
-      transaction do
-        update!(status: :cancelled, completed_at: Time.current, error_message: reason)
-        input_requests.pending.update_all(status: InputRequest.statuses[:cancelled], updated_at: Time.current)
+      now = Time.current
+      cancelled = transaction do
+        changed = self.class.where(id: id, status: self.class.statuses.values_at(*CANCELLABLE_STATUSES))
+          .update_all(status: self.class.statuses[:cancelled], completed_at: now, error_message: reason, updated_at: now)
+        next false if changed.zero?
+
+        input_requests.pending.update_all(status: InputRequest.statuses[:cancelled], updated_at: now)
+        true
       end
-      broadcast_update
+      reload
+      broadcast_update if cancelled
+      cancelled
     end
 
     # Expires the run's overdue requests for input, which fails the run (see

data/app/models/action_agent/evaluation_run.rb

@@ -87,6 +87,48 @@ module ActionAgent
       comparison if comparison.is_a?(Hash) && comparison.present?
     end
 
+    # The bars this run was gated by (ActiveAgent::Evals::PassThresholds),
+    # as the run recorded them when it started or was imported, so a later
+    # change to the catalog does not regrade it; nil for a run whose suite
+    # declared none, one recorded before runs kept them, and a framework
+    # without per-group bars.
+    def pass_thresholds
+      self.class.pass_thresholds_from(scores&.dig("_pass_thresholds"))
+    end
+
+    # A recorded `pass_thresholds` mapping (Report#to_h's: "threshold",
+    # "min_pass_rate", "groups") as PassThresholds, or nil for anything
+    # that is not one.
+    def self.pass_thresholds_from(value)
+      return nil unless value.is_a?(Hash) && defined?(ActiveAgent::Evals::PassThresholds)
+
+      value = value.stringify_keys
+      groups = value["groups"].nil? ? {} : value["groups"]
+      return nil unless groups.is_a?(Hash)
+
+      ActiveAgent::Evals::PassThresholds.new(suite: value.slice(*ActiveAgent::Evals::PassThresholds::SUITE_KEYS), groups: groups)
+    rescue ArgumentError
+      nil
+    end
+
+    # The gates the run's results met or missed (Report#gates, each
+    # `{ "group", "name", "label", "passed", "total", "pass_rate",
+    # "min_pass_rate", "gated", "met" }`), as recorded when it completed;
+    # nil for a run without bars.
+    def gates
+      value = scores&.dig("_gates")
+      value.select { |gate| gate.is_a?(Hash) } if value.is_a?(Array)
+    end
+
+    # Whether every gate was met, as Report#passed? answers: false when no
+    # gate was reached; nil for a run without bars.
+    def gates_met
+      rows = gates
+      return nil if rows.nil?
+
+      rows.any? && rows.all? { |gate| gate["met"] == true }
+    end
+
     # How the report names the judge, the way the suite panel does: the
     # judge the recorded verdict names — unless that is only the pass-rate
     # ranking — else the evaluation's judge model. nil when neither is set,
@@ -307,6 +349,7 @@ module ActionAgent
       kwargs[:judge_usage] = judge_usage_run if judge_usage_run && self.class.report_accepts?(:judge_usage)
       kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
       kwargs[:comparison] = recorded_comparison if recorded_comparison && self.class.report_accepts?(:comparison)
+      kwargs[:pass_thresholds] = pass_thresholds if pass_thresholds && self.class.report_accepts?(:pass_thresholds)
       self.class.report_class.new(**kwargs)
     end
 

data/app/models/action_agent/scenario_product.rb

@@ -21,6 +21,28 @@ module ActionAgent
       value.is_a?(Hash) ? value : {}
     end
 
+    # The product's named criteria lists, as the catalog's `criteria:` gives
+    # them: list name => criterion hashes. A stored value that is not such a
+    # mapping, and a list left with no criterion, read as no list.
+    # @return [Hash{String => Array<Hash>}]
+    def criteria
+      value = self[:criteria]
+      return {} unless value.is_a?(Hash)
+
+      value.each_with_object({}) do |(name, list), lists|
+        entries = Array(list).select { |criterion| criterion.is_a?(Hash) } if list.is_a?(Array)
+        lists[name.to_s] = entries if entries.present?
+      end
+    end
+
+    # The bars the catalog's `pass:` declares (ActiveAgent::Evals::PassThresholds);
+    # a stored value that is not a mapping reads as none.
+    # @return [Hash{String => Object}]
+    def pass
+      value = self[:pass]
+      value.is_a?(Hash) ? value : {}
+    end
+
     # The agent a set of this product runs against when the caller names
     # none: the product's agent, else the project's agent under test, else
     # the owner's agent named by the document.
@@ -36,6 +58,8 @@ module ActionAgent
         "name" => name,
         "description" => description,
         "agent" => agent_name.presence || agent&.name,
+        "criteria" => criteria.presence,
+        "pass" => pass.presence,
         "sets" => sets.map(&:to_document)
       }.merge(metadata).compact
     end

data/app/models/action_agent/scenario_set.rb

@@ -37,6 +37,14 @@ module ActionAgent
       value.is_a?(Hash) ? value : {}
     end
 
+    # The bars the catalog's `pass:` declares (ActiveAgent::Evals::PassThresholds);
+    # a stored value that is not a mapping reads as none.
+    # @return [Hash{String => Object}]
+    def pass
+      value = self[:pass]
+      value.is_a?(Hash) ? value : {}
+    end
+
     # The evaluation's name: the catalog, product and set keys.
     def evaluation_name
       "#{catalog.key}/#{product.key}/#{key}".first(EvaluationReportImport::MAX_EVALUATION_NAME)
@@ -117,6 +125,7 @@ module ActionAgent
         "description" => description,
         "judge" => judge.presence,
         "criteria" => criteria.presence,
+        "pass" => pass.presence,
         "scenarios" => scenarios.map(&:to_document)
       }.merge(metadata).compact
     end

data/app/serializers/action_agent/evaluation_serializer.rb

@@ -81,7 +81,9 @@ module ActionAgent
     # checkout sandbox the run replayed against, if any: its session id and
     # checkout, never its token. `agent_version` is the version of the
     # agent the run scored, and `version_state` whether that is the agent's
-    # current version ("current", "earlier" or "unrecorded").
+    # current version ("current", "earlier" or "unrecorded"). `gates` are the
+    # per-group pass gates the run was held to (EvaluationRun#gates) and
+    # `gates_met` whether it met every one; both nil for a run without bars.
     def run_summary(run, number: nil)
       {
         id: run.id,
@@ -95,7 +97,9 @@ module ActionAgent
         created_at: run.created_at.iso8601,
         sandbox: run.sandbox,
         agent_version: run.agent_version_summary,
-        version_state: run.evaluation.standing_info.version_state(run)
+        version_state: run.evaluation.standing_info.version_state(run),
+        gates: run.gates,
+        gates_met: run.gates_met
       }
     end
 

data/app/services/action_agent/evaluation_report_import.rb

@@ -95,6 +95,10 @@ module ActionAgent
     JUDGE_USAGE_KEYS = %w[calls input_tokens output_tokens cost model by_kind source].freeze
     JUDGE_USAGE_SOURCES = %w[reported traces meter].freeze
     MAX_JUDGE_KINDS = 10
+    # Groups a report's `pass_thresholds` may name; a group holds at least
+    # one result, so no report needs more.
+    MAX_PASS_GROUPS = MAX_RESULTS
+    PASS_THRESHOLD_KEYS = %w[threshold min_pass_rate groups].freeze
     SCOPE_PATTERN = %r{\A[\w .:/@-]{1,100}\z}
     STATUSES = %w[passed failed errored].freeze
     # The envelope statuses a delivery may name.
@@ -674,10 +678,21 @@ module ActionAgent
         "_verdict" => report["verdict"],
         "_selection" => selection,
         "_metadata" => metadata,
-        "_judge_usage_run" => bounded_judge_usage(report.dig("judge_usage", "run"))
+        "_judge_usage_run" => bounded_judge_usage(report.dig("judge_usage", "run")),
+        "_pass_thresholds" => recorded_pass_thresholds
       }.compact.merge("_judge_label" => judge_label)
     end
 
+    # The bars the report was gated by, as PassThresholds normalizes them
+    # (validate_pass_thresholds! has refused anything else), so the rebuilt
+    # report gates the stored results the way the application's did. nil
+    # for a report whose suite declared none.
+    def recorded_pass_thresholds
+      return nil if report["pass_thresholds"].nil?
+
+      EvaluationRun.pass_thresholds_from(report["pass_thresholds"])&.to_h
+    end
+
     # The run's scores in the shape the Evaluations view renders
     # (ScenarioEvaluationRunner#scores_for), summarized from the stored
     # results as the application reported them — no cost is estimated here,
@@ -689,10 +704,14 @@ module ActionAgent
     def summarized_scores(run)
       rebuilt = run.to_report(estimate: false)
       comparison = rebuilt.comparison if rebuilt.respond_to?(:comparison)
+      # The gates are worked out from the stored results as well: the
+      # report's own "gates" are not taken as the verdict.
+      gates = rebuilt.gates.map(&:to_h) if rebuilt.respond_to?(:pass_thresholds) && rebuilt.pass_thresholds
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
         "_recommendations" => rebuilt.recommendations,
-        "_comparison" => comparison
+        "_comparison" => comparison,
+        "_gates" => gates
       ).compact.merge(recorded_scores)
     end
 
@@ -725,6 +744,7 @@ module ActionAgent
       judge_trace_ids!(metadata["judge_trace_ids"])
       string!(report["judge"], "report.judge", 200)
       validate_release!
+      validate_pass_thresholds!
       validate_judge_usage!(report.dig("judge_usage", "run"), "report.judge_usage.run") if report["judge_usage"].is_a?(Hash)
       optional_object!(report["judge_usage"], "report.judge_usage")
       object!(report["models"], "report.models")
@@ -856,6 +876,33 @@ module ActionAgent
       string!(release["label"], "report.release.label", 200)
     end
 
+    # The bars the suite declared, as Report#to_h writes them: "threshold"
+    # and "min_pass_rate" from 0 to 1, and "groups", each group's
+    # "min_pass_rate" and "gated", by group key. Checked by PassThresholds
+    # itself, so the import refuses what the framework would.
+    def validate_pass_thresholds!
+      bars = report["pass_thresholds"]
+      return if bars.nil?
+
+      object!(bars, "report.pass_thresholds")
+      unknown = bars.keys - PASS_THRESHOLD_KEYS
+      raise Invalid, "report.pass_thresholds has unknown keys #{unknown.join(', ')}" if unknown.any?
+
+      groups = bars["groups"]
+      optional_object!(groups, "report.pass_thresholds.groups")
+      if groups && groups.size > MAX_PASS_GROUPS
+        raise Invalid, "report.pass_thresholds.groups must name at most #{MAX_PASS_GROUPS} groups"
+      end
+
+      (groups || {}).each_key { |key| string!(key, "report.pass_thresholds.groups key", 200) }
+      # A framework without per-group bars stores none, as before.
+      return unless defined?(ActiveAgent::Evals::PassThresholds)
+
+      ActiveAgent::Evals::PassThresholds.new(suite: bars.slice(*ActiveAgent::Evals::PassThresholds::SUITE_KEYS), groups: groups || {})
+    rescue ArgumentError => e
+      raise Invalid, "report.pass_thresholds: #{e.message}"
+    end
+
     # What the judge spent, as ActiveAgent::Evals::Report writes it per
     # result and for the run.
     def validate_judge_usage!(usage, name)

data/app/services/action_agent/scenario_catalog_import.rb

@@ -181,6 +181,10 @@ module ActionAgent
           metadata: entry.metadata, agent: resolve_agent(entry) || (merge? ? product.agent : nil),
           project: resolve_project(entry) || (merge? ? product.project : nil)
         )
+        # Until add_catalog_product_criteria runs, imports keep working without the lists.
+        product.criteria = entry.criteria if product.has_attribute?(:criteria)
+        # Until add_catalog_pass_bars runs, imports keep working without the bars.
+        product.pass = entry.pass if product.has_attribute?(:pass)
         product.position = position_for(product, entry.position, offset)
         product.save!
         replace_sets(product, entry)
@@ -197,6 +201,7 @@ module ActionAgent
           name: set_entry.name, description: set_entry.description, judge: set_entry.judge,
           criteria: set_entry.criteria, metadata: set_entry.metadata
         )
+        set.pass = set_entry.pass if set.has_attribute?(:pass)
         set.position = position_for(set, set_entry.position, offset)
         set.save!
         replace_scenarios(set, set_entry)

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -22,6 +22,11 @@ module ActionAgent
   #                        decisive (ActiveAgent::Evals::Comparison)
   #   "_selection"       — the scenarios and models this run covered, and
   #                        the checkout sandbox it replayed against, if any
+  #   "_pass_thresholds" — for an evaluation of a catalog set that declares
+  #                        `pass` bars: the bars, recorded as the run starts,
+  #                        so every replay and the completion gate by the same
+  #                        ones however the catalog changes meanwhile
+  #   "_gates"           — with them: each group's gate per model, met or not
   #
   # A selection's `sandbox_id` names a checkout sandbox whose app runtime
   # every replay reaches beside the agent's own MCP servers, as if the agent
@@ -81,6 +86,7 @@ module ActionAgent
       specs = model_specs
       run = @run || @evaluation.evaluation_runs.create!(status: :pending)
       run.update!(status: :running, selection: selection_summary(scenarios, specs))
+      record_pass_thresholds!(run)
       ensure_sandbox_live!
 
       if scenarios.empty?
@@ -125,7 +131,7 @@ module ActionAgent
         adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: metered)
       else
         ensure_judge_defined_kpis! if @evaluation.judge_defined?
-        default_report(tasks, specs, on_result)
+        default_report(tasks, specs, on_result, run)
       end
       raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
       reported = report.results.map { |result| [ result.scenario.key, result.label ] }
@@ -453,7 +459,7 @@ module ActionAgent
         available_tools: fan_out_state(run).fetch("tools") { tool_roster },
         instructions: @evaluation.agent.instructions,
         agent_name: @evaluation.agent.name,
-        threshold: PASS_THRESHOLD,
+        **gating(run.pass_thresholds, Evals::Runner),
         replay: ->(*) { raise ArgumentError, "a fanned-out result is scored from its stored replay" },
         refine_gate: -> { reserve_refinement!(run) }
       )
@@ -513,7 +519,7 @@ module ActionAgent
         )
       end
       report = Evals::Report.new(results: results, models: specs.values, judge: evals_judge,
-                                 instructions: @evaluation.agent.instructions, threshold: PASS_THRESHOLD)
+                                 instructions: @evaluation.agent.instructions, **gating(run.pass_thresholds, Evals::Report))
       state = fan_out_state(run)
 
       # The other jobs' judge calls go into this runner's meter before the
@@ -559,7 +565,7 @@ module ActionAgent
       error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
     end
 
-    def default_report(tasks, specs, on_result)
+    def default_report(tasks, specs, on_result, run)
       Evals::Runner.new(
         scenarios: tasks,
         models: specs,
@@ -568,7 +574,7 @@ module ActionAgent
         available_tools: tool_roster,
         instructions: @evaluation.agent.instructions,
         agent_name: @evaluation.agent.name,
-        threshold: PASS_THRESHOLD,
+        **gating(run.pass_thresholds, Evals::Runner),
         replay: ->(scenario, spec) { replay(scenario, spec) },
         on_result: on_result
       ).call
@@ -827,6 +833,7 @@ module ActionAgent
       comparison = report.comparison if report.respond_to?(:comparison)
       scores["_comparison"] = comparison if comparison
       scores["_selection"] = run.selection
+      record_gates(scores, report, run.pass_thresholds)
       scores["_metadata"] = report.metadata
       scores["_judge_label"] = report.judge_label || report.judge&.label
       # The judge's own spend, apart from the replays' (which the results
@@ -839,6 +846,60 @@ module ActionAgent
       scores
     end
 
+    # The bars a host adapter's report did not carry are the run's own, so
+    # the run is gated however its replays ran.
+    def record_gates(scores, report, recorded)
+      thresholds = (report.pass_thresholds if report.respond_to?(:pass_thresholds)) || recorded
+      return if thresholds.nil?
+
+      scores["_pass_thresholds"] = thresholds.to_h
+      scores["_gates"] = report.gates(thresholds).map(&:to_h)
+    end
+
+    # --- pass bars ----------------------------------------------------------
+
+    # The score keyword, and the bars when the run has them and the
+    # installed framework's +klass+ takes them: the bars' threshold is the
+    # catalog's or PASS_THRESHOLD, the score the dashboard always used.
+    def gating(thresholds, klass)
+      return { threshold: PASS_THRESHOLD } if thresholds.nil? || !accepts_keyword?(klass, :pass_thresholds)
+
+      { threshold: thresholds.threshold, pass_thresholds: thresholds }
+    end
+
+    def accepts_keyword?(klass, keyword)
+      klass.instance_method(:initialize).parameters.any? { |type, name| name == keyword && %i[key keyreq].include?(type) }
+    end
+
+    # Records the bars of the catalog set this evaluation runs as, once,
+    # as the run starts: a fanned-out replay reads them from the run.
+    def record_pass_thresholds!(run)
+      thresholds = catalog_pass_thresholds
+      return if thresholds.nil?
+
+      run.update!(scores: (run.scores.is_a?(Hash) ? run.scores : {}).merge("_pass_thresholds" => thresholds.to_h))
+    end
+
+    # The product's bars as the suite's and the set's as its group's, as
+    # Catalog#suite_for gives them, for an evaluation a catalog set
+    # materialized (ScenarioSet#catalog_reference) when the set, under the
+    # owner's catalogs, still has the keys the reference names; nil when
+    # it declares none, and on a framework without per-group bars.
+    def catalog_pass_thresholds
+      reference = @evaluation.config.to_h["catalog"]
+      return nil unless reference.is_a?(Hash) && defined?(Evals::PassThresholds)
+
+      set = ScenarioSet.joins(:product).includes(:product)
+        .where(ScenarioProduct.table_name => { scenario_catalog_id: ScenarioCatalog.for_owner(owner).select(:id) })
+        .find_by(id: reference["set_id"], key: reference["set_key"].to_s)
+      return nil if set.nil? || set.product.key != reference["product_key"].to_s
+      return nil if set.product.pass.empty? && set.pass.empty?
+
+      Evals::PassThresholds.new(suite: set.product.pass, groups: { set.key => set.pass })
+    rescue ArgumentError
+      nil
+    end
+
     # Records who judged once the calls are in. A judge with no model pinned
     # started out labelled by its provider; it is renamed after the model
     # that answered, here and on the verdict it wrote.

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.9.7"
+  VERSION = "1.9.8"
 end

data/lib/generators/action_agent/templates/migrations/034_add_catalog_product_criteria.rb.erb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AddCatalogProductCriteria < ActiveRecord::Migration<%= migration_version %>
+  def up
+    return unless table_exists?(products_table)
+    return if column_exists?(products_table, :criteria)
+
+    type = connection.adapter_name.to_s.downcase.include?("postgres") ? :jsonb : :json
+    add_column products_table, :criteria, type
+  end
+
+  def down
+    return unless table_exists?(products_table)
+    return unless column_exists?(products_table, :criteria)
+
+    remove_column products_table, :criteria
+  end
+
+  private
+
+  def products_table
+    "#{ActionAgent.table_name_prefix}scenario_products"
+  end
+end

data/lib/generators/action_agent/templates/migrations/035_add_catalog_pass_bars.rb.erb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class AddCatalogPassBars < ActiveRecord::Migration<%= migration_version %>
+  def up
+    type = connection.adapter_name.to_s.downcase.include?("postgres") ? :jsonb : :json
+    catalog_tables.each do |table|
+      next unless table_exists?(table)
+      next if column_exists?(table, :pass)
+
+      add_column table, :pass, type
+    end
+  end
+
+  def down
+    catalog_tables.each do |table|
+      next unless table_exists?(table)
+      next unless column_exists?(table, :pass)
+
+      remove_column table, :pass
+    end
+  end
+
+  private
+
+  def catalog_tables
+    %w[scenario_products scenario_sets].map { |table| "#{ActionAgent.table_name_prefix}#{table}" }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.9.7
+  version: 1.9.8
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-11-05 00:00:00.000000000 Z
+date: 2026-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -383,6 +383,8 @@ files:
 - lib/generators/action_agent/templates/migrations/030_add_evaluation_scoring_leases.rb.erb
 - lib/generators/action_agent/templates/migrations/031_add_catalog_scenario_metadata.rb.erb
 - lib/generators/action_agent/templates/migrations/032_create_active_agent_mcp_server_configs.rb.erb
+- lib/generators/action_agent/templates/migrations/034_add_catalog_product_criteria.rb.erb
+- lib/generators/action_agent/templates/migrations/035_add_catalog_pass_bars.rb.erb
 - lib/tasks/action_agent.rake
 - lib/tasks/browser.rake
 - lib/tasks/claude_code.rake