actionagent 1.8.1 → 1.9.0

data/app/services/action_agent/evaluation_fix.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # A bounded, server-built brief for one persisted fix card. Request bodies
+  # choose an existing card and a subset of its scope; they cannot inject a
+  # different prompt, agent, scenario or model into the automatic fix loop.
+  class EvaluationFix
+    class Invalid < StandardError; end
+    attr_reader :run, :item
+
+    def initialize(run, requested)
+      @run = run
+      raise Invalid, "Only completed scenario runs have implementable fixes" unless run.complete? && run.evaluation.scenario_suite?
+      requested = requested.to_h.deep_stringify_keys
+      original = EvaluationSerializer.fix_items(run).find do |candidate|
+        candidate = candidate.deep_stringify_keys
+        candidate["kind"] == requested["kind"] &&
+          (requested["kind"] == "instruction" ? candidate["quote"] == requested["quote"] : candidate["fault"] == requested["fault"])
+      end
+      raise Invalid, "This fix card no longer exists on the selected run" unless original && %w[fault instruction].include?(requested["kind"])
+
+      @item = original.deep_stringify_keys.slice("kind", "fault", "quote", "recommendation", "scenario_keys", "models")
+      keys = Array(requested["scenario_keys"])
+      allowed_models = Array(@item["models"]).presence || model_labels
+      models = Array(requested["models"]).presence || allowed_models
+      unless keys.any? && (keys - Array(@item["scenario_keys"])).empty? && models.any? && (models - allowed_models).empty?
+        raise Invalid, "Choose scenarios and models from this fix card"
+      end
+      @item.merge!("scenario_keys" => keys.uniq, "models" => models.uniq)
+      @item["scenarios"] = run.scenario_results.includes(:scenario).filter_map do |result|
+        snapshot = result.evaluated_scenario.slice("key", "prompt", "expectations", "notes", "enabled")
+        snapshot if keys.include?(snapshot["key"])
+      end.uniq
+      self.class.check_scenarios!(run.evaluation, @item)
+    end
+
+    def self.check_scenarios!(evaluation, item)
+      Array(item["scenarios"]).each do |snapshot|
+        current = evaluation.scenarios.find_by(key: snapshot.fetch("key"))
+        unless current&.enabled? && current.as_json_summary.stringify_keys.slice(*snapshot.keys) == snapshot
+          raise Invalid, "The selected scenarios changed since this run. Run the evaluation again before implementing a fix."
+        end
+      end
+    end
+
+    def model_labels
+      run.scenario_results.map { |result| [ result.provider, result.model ].compact_blank.join("/") }.uniq
+    end
+
+    # How much of each piece of evidence the brief quotes, at full size. A
+    # card with many scenarios and models, or a retry, shrinks them all
+    # (see #prompt) rather than refusing the card.
+    EXCERPTS = { prompt: 1000, expectations: 1000, diagnosis: 1200, answer: 1000,
+      previous_result: 1500, previous_diff: 5000, previous_results: 3000 }.freeze
+    SCALES = [ 1, 0.5, 0.25, 0.1 ].freeze
+
+    def prompt(previous: nil)
+      SCALES.each do |scale|
+        text = brief(previous, EXCERPTS.transform_values { |size| (size * scale).to_i })
+        return text if text.length <= CodeSession::MAX_PROMPT_CHARACTERS
+      end
+      raise Invalid, "This fix has too much context for one session; run the evaluation on fewer scenarios or models"
+    end
+
+    private
+
+    def brief(previous, limit)
+      agent = run.evaluation.agent
+      name = agent.agent_class_name.presence || agent.telemetry_agent_class
+      path = name.to_s.underscore
+      lines = [ "# Implement one evaluation fix", "", "Agent: #{name}", "Evaluation: #{run.evaluation.name} · run #{run.id}",
+        "", "## What to fix", item["fault"], item["recommendation"], item["quote"],
+        "", "## Where to look", "- app/agents/#{path}.rb and its tools.",
+        "- app/views/agents/#{path.delete_suffix('_agent')}/ and app/views/#{path}/ for instructions and prompts.",
+        "- Follow the checkout's own instructions and locate the source of a synced agent before editing.",
+        "", "## Failing scenarios" ]
+      run.scenario_results.includes(:scenario).each do |result|
+        scenario = result.evaluated_scenario
+        next unless item["scenario_keys"].include?(scenario["key"])
+        label = [ result.provider, result.model ].compact_blank.join("/")
+        next unless item["models"].include?(label) || item["models"].include?(result.model)
+        lines += [ "", "### #{scenario['key']} · #{label}",
+          "Prompt: #{scenario['prompt'].to_s.truncate(limit[:prompt])}", "Expectations: #{scenario['expectations'].to_json.truncate(limit[:expectations])}",
+          "Before: #{result.status} · score #{result.score} · fault #{result.fault}",
+          "Diagnosis: #{result.diagnosis.to_json.truncate(limit[:diagnosis])}", "Answer: #{result.output.to_s.truncate(limit[:answer])}" ]
+      end
+      if previous
+        lines += [ "", "## Previous attempt", previous.result.to_s.truncate(limit[:previous_result]),
+          "Previous diff:", previous.diff.to_s.truncate(limit[:previous_diff]),
+          "Verification results: #{EvaluationFixComparison.new(previous).rows.to_json.truncate(limit[:previous_results])}" ]
+      end
+      lines += [ "", "## Verify", "Change the agent, not the scenarios or their expectations.",
+        "Run the relevant checkout tests. Do not commit or push.",
+        "The dashboard will re-run these exact scenarios and models against this sandbox when you finish.",
+        "Do not call dashboard URLs or dashboard MCP tools from this sandbox.",
+        "Never read or copy the sandbox's Claude configuration or credentials.",
+        "Summarize the change and any remaining limitations." ]
+      lines.compact.join("\n")
+    end
+  end
+end

data/app/services/action_agent/evaluation_fix_comparison.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  class EvaluationFixComparison
+    def initialize(session)
+      @session = session
+    end
+
+    def self.change(before, after)
+      return "pending" unless after
+      return after.passed? ? "added_pass" : "added_fail" unless before
+      return after.passed? ? "unchanged_pass" : "regressed" if before.passed?
+      after.passed? ? "fixed" : "still_failing"
+    end
+
+    def rows
+      before = indexed(@session.evaluation_run)
+      after = indexed(@session.verification_run)
+      (before.keys | after.keys).map do |key|
+        first, last = before[key], after[key]
+        { scenario_key: key[0], model: key[1], change: self.class.change(first, last),
+          before: first&.status, after: last&.status, before_score: first&.score&.to_f, after_score: last&.score&.to_f,
+          score_change: first&.score && last&.score ? (last.score - first.score).to_f.round(3) : nil,
+          before_fault: first&.fault, fault: last&.fault }
+      end
+    end
+
+    def summary
+      { run_id: @session.verification_run_id, status: @session.verification_run&.status,
+        error: @session.verification_error || @session.verification_run&.error_message,
+        counts: rows.group_by { |row| row[:change] }.transform_values(&:size), rows: rows }
+    end
+
+    # The before/after section a fix's draft pull request carries, cut to
+    # +limit+ characters from the end (the session summary goes first), so
+    # appending it never pushes a body over DraftPullRequest's maximum.
+    def pull_request_body(mount:, limit: DraftPullRequest::MAX_BODY_CHARACTERS)
+      base = @session.evaluation_run
+      title = @session.fix_item["quote"].presence || @session.fix_item["fault"].to_s.humanize
+      lines = [ "## Evaluation fix", title, "",
+        "[Original run](#{mount}/evaluations/#{base.evaluation_id}/runs/#{base.id})",
+        "[Verification run](#{mount}/evaluations/#{base.evaluation_id}/runs/#{@session.verification_run_id})", "",
+        "| Scenario / model | Before | After | Score change | Fault |", "| --- | --- | --- | --- | --- |" ]
+      rows.each do |row|
+        cells = [ "#{row[:scenario_key]} / #{row[:model]}", row[:before], row[:after] || "pending", row[:score_change], row[:fault] || "—" ]
+        lines << "| #{cells.map { |cell| cell.to_s.gsub('|', '\\|').gsub(/[\r\n]/, ' ') }.join(' | ')} |"
+      end
+      lines += [ "", "## Session summary", @session.result.to_s, "", "Code session ##{@session.id}. Review the diff and full-suite regressions before merging." ]
+      SecretScrubber.scrub(lines.join("\n"), @session.secrets).truncate([ limit, 0 ].max, omission: "\n…")
+    end
+
+    private
+
+    def indexed(run)
+      return {} unless run
+      item = @session.fix_item || {}
+      run.scenario_results.includes(:scenario).each_with_object({}) do |result, index|
+        key = result.evaluated_scenario["key"]
+        model = [ result.provider, result.model ].compact_blank.join("/")
+        next unless Array(item["scenario_keys"]).include?(key) &&
+          (Array(item["models"]).include?(model) || Array(item["models"]).include?(result.model))
+        index[[ key, model ]] = result
+      end
+    end
+  end
+end

data/app/services/action_agent/evaluation_runner_service.rb

@@ -594,13 +594,12 @@ module ActionAgent
 
     # The evaluated agent owner's credential for +name+ (Settings ->
     # Provider API Keys, or whatever the host app resolves); preferred over
-    # the host's config/active_agent.yml credentials for the judge.
+    # the host's config/active_agent.yml credentials for the judge. Resolved
+    # with no actor: the judge scores for the organization, so it never runs
+    # on a member's personal key.
     def owner_provider_options(name)
       @owner_provider_options ||= {}
-      @owner_provider_options[name.to_s] ||= begin
-        from_host = ActionAgent.provider_credentials(owner, name.to_s)
-        from_host.presence || ProviderKey.for_owner(owner).find_by(provider: name.to_s)&.generation_options || {}
-      end
+      @owner_provider_options[name.to_s] ||= ProviderCredentials.resolve(owner: owner, actor: nil, provider: name).options
     end
 
     def owner

data/app/services/action_agent/evaluation_standing.rb

@@ -17,16 +17,19 @@ module ActionAgent
   #   archived   — the evaluation is archived (Evaluation#archive!)
   #   none       — no complete run
   #
-  # Only edits the model can see count as a new version: the instructions,
-  # the action prompts, the tools, the MCP servers, the model config and
-  # the response format. A dashboard edit to the agent's appearance cuts an
-  # AgentVersion like any other edit, but the run before it scored the same
-  # agent and stays current.
+  # Only edits that change how a run goes count as a new version: what the
+  # model is given (the instructions, the action prompts, the tools, the MCP
+  # servers, the model config and the response format), and the tools whose
+  # calls wait for a person's approval, which pause a replay. A dashboard
+  # edit to the agent's appearance cuts an AgentVersion like any other edit,
+  # but the run before it scored the same agent and stays current.
   class EvaluationStanding
     STANDINGS = %w[current stale unrecorded archived none].freeze
     VERSION_STATES = %w[current earlier unrecorded].freeze
     # The configuration snapshot keys the model is given.
     MODEL_FACING = %w[instructions action_prompts tools mcp_servers model_config response_format].freeze
+    # The configuration snapshot keys that change how a run goes.
+    SCORED = (MODEL_FACING + %w[approval_required_tools]).freeze
 
     # Gives every evaluation its standing with one query per table rather
     # than one per evaluation: the agents' latest versions and whether each
@@ -95,7 +98,7 @@ module ActionAgent
 
       latest = latest_version
       return "current" if latest.nil? || latest.id == version.id
-      return "current" if model_facing(version) == model_facing(latest)
+      return "current" if scored_configuration(version) == scored_configuration(latest)
 
       "earlier"
     end
@@ -137,9 +140,13 @@ module ActionAgent
       @has_releases = evaluation.agent&.agent_versions&.releases&.exists? || false
     end
 
-    def model_facing(version)
+    # A snapshot taken before an agent had an approval list reads as an empty
+    # one, so the list's arrival alone cuts no new version.
+    def scored_configuration(version)
       snapshot = version.configuration_snapshot.to_h.stringify_keys
-      MODEL_FACING.to_h { |key| [ key, snapshot[key] ] }
+      configuration = SCORED.to_h { |key| [ key, snapshot[key] ] }
+      configuration["approval_required_tools"] = Array(configuration["approval_required_tools"])
+      configuration
     end
   end
 end