actionagent 1.2.2 → 1.5.0

data/app/models/action_agent/evaluation_scenario.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # One user-authored task an evaluation replays through its agent: the
+  # message a user would send, the group of related tasks it belongs to, and
+  # what a passing answer is expected to do.
+  #
+  # `expectations` holds the scenario-level checks ScenarioEvaluationRunner
+  # scores in addition to the evaluation's criteria:
+  #
+  #   "tools"        — tool names the agent is expected to call (any one of them)
+  #   "contains"     — substrings or patterns the answer must include
+  #   "not_contains" — substrings or patterns the answer must avoid
+  #
+  # `key` is stable within the suite ("blame_3"), so results of successive
+  # runs line up by scenario even after the suite is re-imported.
+  class EvaluationScenario < ApplicationRecord
+    belongs_to :evaluation
+    has_many :results, class_name: "EvaluationScenarioResult", foreign_key: :evaluation_scenario_id,
+      dependent: :destroy, inverse_of: :scenario
+
+    validates :key, presence: true, uniqueness: { scope: :evaluation_id }
+    validates :prompt, presence: true
+
+    scope :enabled, -> { where(enabled: true) }
+    scope :ordered, -> { order(:position, :id) }
+    scope :in_group, ->(group) { where(group: group) }
+
+    def expectations
+      value = super
+      value.is_a?(Hash) ? value : {}
+    end
+
+    def expected_tools
+      Array(expectations["tools"]).map(&:to_s).reject(&:blank?)
+    end
+
+    def expected_patterns
+      Array(expectations["contains"]).map(&:to_s).reject(&:blank?)
+    end
+
+    def forbidden_patterns
+      Array(expectations["not_contains"]).map(&:to_s).reject(&:blank?)
+    end
+
+    def as_json_summary
+      {
+        id: id,
+        key: key,
+        group: group,
+        prompt: prompt,
+        notes: notes,
+        expectations: expectations,
+        position: position,
+        enabled: enabled
+      }
+    end
+  end
+end

data/app/models/action_agent/evaluation_scenario_result.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # What one scenario produced under one model in one evaluation run: the
+  # AgentRun that replayed it, the answer, the tools it called, its score per
+  # criterion, and — when it fell short — the fault and the recommended fix.
+  #
+  # `fault` is one of FAULTS; `diagnosis` carries the evidence behind it and,
+  # when a judge was available, its suggested tool or instruction change.
+  class EvaluationScenarioResult < ApplicationRecord
+    belongs_to :evaluation_run
+    belongs_to :scenario, class_name: "EvaluationScenario", foreign_key: :evaluation_scenario_id, inverse_of: :results
+    belongs_to :agent_run, optional: true
+
+    enum :status, { pending: 0, passed: 1, failed: 2, errored: 3 }
+
+    # Why a scenario did not pass, from the most mechanical cause to the
+    # least; ActiveAgent::Evals::Diagnosis assigns exactly one per failing result.
+    FAULTS = ActiveAgent::Evals::Diagnosis::FAULTS
+
+    validates :model, presence: true
+    validates :fault, inclusion: { in: FAULTS }, allow_nil: true
+
+    scope :for_model, ->(model) { where(model: model) }
+    scope :faulted, -> { where.not(fault: nil) }
+
+    def tool_calls
+      value = super
+      value.is_a?(Array) ? value : []
+    end
+
+    def tool_names
+      tool_calls.filter_map { |call| call.is_a?(Hash) ? (call["name"] || call[:name]) : call }.map(&:to_s)
+    end
+
+    def tool_errors
+      tool_calls.select { |call| call.is_a?(Hash) && (call["error"] || call[:error]) }
+    end
+
+    # Adapter-provided correlation and context are stored alongside diagnosis
+    # in a reserved JSON key, leaving the public diagnosis contract unchanged.
+    def replay_metadata
+      value = diagnosis&.dig("_replay_metadata")
+      value.is_a?(Hash) ? value : {}
+    end
+
+    def evaluation_diagnosis
+      (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot")
+    end
+
+    # A catalog can be refreshed without changing what an earlier run asked
+    # or expected. Older results did not record this snapshot.
+    def evaluated_scenario
+      snapshot = diagnosis&.dig("_scenario_snapshot")
+      snapshot.is_a?(Hash) ? snapshot : scenario.as_json_summary.stringify_keys
+    end
+
+    def as_json_summary
+      {
+        id: id,
+        scenario_id: evaluation_scenario_id,
+        scenario_key: evaluated_scenario["key"],
+        group: evaluated_scenario["group"],
+        prompt: evaluated_scenario["prompt"],
+        scenario: evaluated_scenario,
+        model: model,
+        provider: provider,
+        status: status,
+        score: score,
+        scores: scores,
+        output: output,
+        tool_calls: tool_calls,
+        duration_ms: duration_ms,
+        input_tokens: input_tokens,
+        output_tokens: output_tokens,
+        cost: cost&.to_f,
+        fault: fault,
+        recommendation: recommendation,
+        diagnosis: evaluation_diagnosis,
+        metadata: replay_metadata,
+        error_message: error_message,
+        agent_run_id: agent_run_id
+      }
+    end
+  end
+end

data/app/models/action_agent/recording_action.rb

@@ -75,14 +75,23 @@ module ActionAgent
       }
     end
 
-    private
-
+    # The value with passwords, card numbers and the like masked. Public so
+    # every read path — /actions, the show timeline, and the export
+    # cassette — serializes through the one redaction rule.
     def redacted_value
       return value unless should_redact?
 
       "[REDACTED]"
     end
 
+    # Metadata with the sensitive keys removed. Public for the same reason.
+    def safe_metadata
+      # Remove any sensitive data from metadata
+      metadata.except("password", "credit_card", "cvv", "ssn")
+    end
+
+    private
+
     def should_redact?
       return false unless value.present?
 
@@ -102,11 +111,6 @@ module ActionAgent
       selector_is_sensitive || value_is_sensitive
     end
 
-    def safe_metadata
-      # Remove any sensitive data from metadata
-      metadata.except("password", "credit_card", "cvv", "ssn")
-    end
-
     def extract_url
       case action_type
       when "navigate"

data/app/models/action_agent/sandbox_session.rb

@@ -48,7 +48,7 @@ module ActionAgent
     # Unknown keys are dropped rather than raising — a session outlives a
     # catalog edit.
     def mcp_catalog_entries
-      Array(mcp_servers).filter_map { |key| McpCatalog.find(key) }
+      Array(mcp_servers).filter_map { |key| MCPCatalog.find(key) }
     end
 
     # Check if session is still valid

data/app/models/action_agent/session_recording.rb

@@ -31,20 +31,30 @@ module ActionAgent
       name&.start_with?("user_takeover_")
     end
 
-    # Start a new recording session
-    def self.start!(agent_run: nil, sandbox_session: nil, name: nil)
-      create!(
+    # Start a new recording session.
+    #
+    # The owner column is written here, at creation: the index and recent
+    # endpoints scope through it, and a recording nothing ever stamped was
+    # invisible in the list to the very person who made it. An explicit
+    # +owner+ wins; otherwise the recording inherits the owner of the sandbox
+    # or agent it records. Ownable#owner= is a no-op in a single-user
+    # install, where nothing is owned.
+    def self.start!(agent_run: nil, sandbox_session: nil, name: nil, owner: nil)
+      recording = new(
         agent_run: agent_run,
         sandbox_session: sandbox_session,
         name: name || generate_name(agent_run, sandbox_session),
         status: :recording,
         metadata: { started_at: Time.current.iso8601 }
       )
+      recording.owner = owner || inherited_owner(agent_run, sandbox_session)
+      recording.save!
+      recording
     end
 
     # Start a user takeover session (for lander demo analytics)
-    def self.start_user_session!(visitor_id: nil, parent_demo_id: nil, page_url: nil)
-      create!(
+    def self.start_user_session!(visitor_id: nil, parent_demo_id: nil, page_url: nil, owner: nil)
+      recording = new(
         name: "user_takeover_#{Time.current.strftime('%Y%m%d_%H%M%S')}_#{SecureRandom.hex(4)}",
         status: :recording,
         metadata: {
@@ -56,6 +66,16 @@ module ActionAgent
           user_agent: nil # Will be set from request
         }
       )
+      recording.owner = owner
+      recording.save!
+      recording
+    end
+
+    # Whoever owns the sandbox or agent a recording is made against. Both
+    # models declare the same owner candidates as this one, so the record
+    # they hand back is of the class this install owns things through.
+    def self.inherited_owner(agent_run, sandbox_session)
+      sandbox_session&.owner || agent_run&.agent&.owner
     end
 
     # Record a browser action
@@ -112,7 +132,10 @@ module ActionAgent
       )
     end
 
-    # Get timeline data for playback
+    # Get timeline data for playback. Values and metadata go through the
+    # same redaction the /actions endpoint applies: this is what #show
+    # renders, and it used to hand back the cleartext password that
+    # /actions had just redacted for the same action.
     def timeline
       recording_actions.order(:sequence).map do |action|
         {
@@ -121,9 +144,9 @@ module ActionAgent
           sequence: action.sequence,
           timestamp_ms: action.timestamp_ms,
           selector: action.selector,
-          value: action.value,
+          value: action.redacted_value,
           screenshot_key: action.screenshot_key,
-          metadata: action.metadata
+          metadata: action.safe_metadata
         }
       end
     end

data/app/models/action_agent/telemetry_trace.rb

@@ -81,7 +81,110 @@ module ActionAgent
       end
     end
 
-    def self.create_from_payload(trace, sdk_info = {}, account: nil)
+    # The trace columns the Metrics report reads, in the order
+    # pluck_metrics_rows returns them (before the span-derived fields).
+    METRICS_COLUMNS = %i[
+      timestamp agent_class agent_action total_duration_ms status error_message
+      total_input_tokens total_output_tokens
+    ].freeze
+
+    # Plucks one row per trace with everything the Metrics report needs, so
+    # a window is read once:
+    #
+    #   [timestamp, agent_class, agent_action, total_duration_ms, status,
+    #    error_message, total_input_tokens, total_output_tokens,
+    #    llm_model, llm_provider, tool_calls, tool_errors]
+    #
+    # llm_model and llm_provider come from the first llm span; tool_calls
+    # and tool_errors count the trace's tool spans and those whose status is
+    # ERROR. Same split as pluck_with_llm_model: PostgreSQL digs into the
+    # spans jsonb in SQL, other adapters read the column back and count in
+    # Ruby.
+    def self.pluck_metrics_rows(scope)
+      if postgres?
+        scope.pluck(
+          *METRICS_COLUMNS,
+          # spans is cast for the same reason as in pluck_with_llm_model:
+          # the column is json on older installs.
+          Arel.sql(
+            "(SELECT s.value -> 'attributes' ->> 'llm.model' " \
+            "FROM jsonb_array_elements(spans::jsonb) AS s " \
+            "WHERE s.value ->> 'type' = 'llm' LIMIT 1)"
+          ),
+          Arel.sql(
+            "(SELECT s.value -> 'attributes' ->> 'llm.provider' " \
+            "FROM jsonb_array_elements(spans::jsonb) AS s " \
+            "WHERE s.value ->> 'type' = 'llm' LIMIT 1)"
+          ),
+          Arel.sql(
+            "(SELECT COUNT(*) FROM jsonb_array_elements(spans::jsonb) AS s " \
+            "WHERE s.value ->> 'type' = 'tool')"
+          ),
+          Arel.sql(
+            "(SELECT COUNT(*) FROM jsonb_array_elements(spans::jsonb) AS s " \
+            "WHERE s.value ->> 'type' = 'tool' AND s.value ->> 'status' = '#{STATUS_ERROR}')"
+          )
+        )
+      else
+        scope.pluck(:spans, *METRICS_COLUMNS).map do |spans, *rest|
+          spans = Array(spans).grep(Hash)
+          llm = spans.find { |span| span["type"].to_s == "llm" }
+          tools = spans.select { |span| span["type"].to_s == "tool" }
+          [
+            *rest,
+            llm&.dig("attributes", "llm.model"),
+            llm&.dig("attributes", "llm.provider"),
+            tools.size,
+            tools.count { |span| span["status"].to_s == STATUS_ERROR }
+          ]
+        end
+      end
+    end
+
+    # Per-tool call count, error count and summed duration over the tool
+    # spans of every trace in +scope+:
+    #
+    #   { "web_search" => { calls: 12, errors: 1, total_ms: 7680.0 }, ... }
+    #
+    # A tool is named by its span's tool.name attribute, or by the span name
+    # minus its "tool." prefix — the same rule as #tool_usage. PostgreSQL
+    # groups in SQL over the unnested spans jsonb; other adapters read the
+    # spans back and tally in Ruby.
+    def self.tool_span_stats(scope)
+      if postgres?
+        name_sql = "COALESCE(s.value -> 'attributes' ->> 'tool.name', " \
+                   "regexp_replace(COALESCE(s.value ->> 'name', ''), '^tool\\.', ''))"
+
+        scope
+          .joins("CROSS JOIN LATERAL jsonb_array_elements(#{table_name}.spans::jsonb) AS s")
+          .where("s.value ->> 'type' = 'tool'")
+          .group(Arel.sql(name_sql))
+          .pluck(
+            Arel.sql(name_sql),
+            Arel.sql("COUNT(*)"),
+            Arel.sql("SUM(CASE WHEN s.value ->> 'status' = '#{STATUS_ERROR}' THEN 1 ELSE 0 END)"),
+            Arel.sql("SUM(COALESCE((s.value ->> 'duration_ms')::float, 0))")
+          )
+          .to_h do |name, calls, errors, total_ms|
+            [ name, { calls: calls.to_i, errors: errors.to_i, total_ms: total_ms.to_f } ]
+          end
+      else
+        scope.pluck(:spans).each_with_object({}) do |spans, stats|
+          Array(spans).each do |span|
+            next unless span.is_a?(Hash) && span["type"].to_s == "tool"
+
+            attributes = span["attributes"] || {}
+            name = attributes["tool.name"] || span["name"].to_s.delete_prefix("tool.")
+            entry = (stats[name] ||= { calls: 0, errors: 0, total_ms: 0.0 })
+            entry[:calls] += 1
+            entry[:errors] += 1 if span["status"].to_s == STATUS_ERROR
+            entry[:total_ms] += span["duration_ms"].to_f
+          end
+        end
+      end
+    end
+
+    def self.create_from_payload(trace, sdk_info = {}, account: nil, agent: nil)
       spans = trace["spans"] || []
       root_span = spans.find { |s| s["parent_span_id"].nil? } || spans.first || {}
 
@@ -135,9 +238,24 @@ module ActionAgent
       # Add account if in multi-tenant mode
       attrs[:account] = account if ActionAgent.multi_tenant? && account
 
+      # A trace the dashboard recorded for one of its own runs belongs to the
+      # agent it ran, which only the caller that ran it may name (+agent+,
+      # passed by AgentExecutionService). Attribute it up front: left to the
+      # registrar, the run's class and action match no authored record —
+      # those carry no service_name or agent_class_name — and every dashboard
+      # run would register an "observed" twin of the agent that produced it.
+      #
+      # Never taken from the trace itself. A payload's resource attributes are
+      # whatever the reporter sent, and ingest is unauthenticated on a
+      # single-tenant install with no ActionAgent.ingest_api_key, so trusting
+      # an id from there would let any reporter bind its traces to any
+      # dashboard-authored agent by guessing a primary key.
+      attrs[:agent_id] = agent&.id
+
       create!(attrs).tap { |record| AgentRegistrar.call(record) }
     end
 
+
     # Sums a span's token counts (used to decide which spans carry the
     # authoritative token data during ingestion).
     #
@@ -193,8 +311,13 @@ module ActionAgent
           duration_ms: span["duration_ms"],
           status: span["status"],
           error: attributes["error.message"],
-          arguments: attributes["tool.input.args"],
-          result: attributes["tool.output.result"]
+          # Two key families: the framework's instrumentation records
+          # tool.input.args / tool.output.result, the activeagents-telemetry
+          # ruby_llm adapter records tool.arguments / tool.result. The
+          # Interactions serializer already reads both; the Tools view's
+          # sample arguments come from here and were blank for adapter traffic.
+          arguments: attributes["tool.input.args"] || attributes["tool.arguments"],
+          result: attributes["tool.output.result"] || attributes["tool.result"]
         }
       end
     end

data/app/models/concerns/action_agent/adapter_aware.rb

@@ -45,6 +45,25 @@ module ActionAgent
 
         Time.find_zone("UTC").parse(value.to_s).to_i
       end
+
+      # Epoch second of the +seconds+-wide bucket holding +value+. Buckets
+      # are aligned to the Unix epoch, so the same timestamp lands in the
+      # same bucket whatever form an adapter produced it in: a Time
+      # (PostgreSQL, or any adapter's type-cast pluck), the UTC string
+      # SQLite and MySQL return from raw SQL, or an epoch number.
+      def bucket_epoch(value, seconds)
+        epoch = value.is_a?(Numeric) ? value.to_i : hour_bucket_epoch(value)
+        epoch - (epoch % seconds.to_i)
+      end
+
+      # Start epochs of the +count+ consecutive +seconds+-wide buckets that
+      # end with the one containing +now+, oldest first. The last bucket is
+      # the live one, so the window these describe is
+      # [starts.first, starts.last + seconds) — it runs a little past now.
+      def bucket_starts(now, seconds, count)
+        last = bucket_epoch(now, seconds)
+        Array.new(count) { |index| last - ((count - 1 - index) * seconds) }
+      end
     end
   end
 end

data/app/models/concerns/action_agent/ownable.rb

@@ -71,16 +71,29 @@ module ActionAgent
     end
 
     # The record's owner under the current configuration, or nil.
+    #
+    # The belongs_to is declared when the class loads, from the
+    # configuration at that moment; an owner model configured afterwards
+    # (a test, or an initializer that ran late) has the column but not the
+    # association, so the foreign key is read directly in that case.
     def owner
       association = self.class.owner_association
-      association && public_send(association)
+      return nil unless association
+      return public_send(association) if respond_to?(association)
+
+      owner_class = ActionAgent.public_send(CLASS_FOR.fetch(association)).safe_constantize
+      owner_id = self[:"#{association}_id"]
+      owner_class.find_by(id: owner_id) if owner_class && owner_id
     end
 
     # Assigns +owner+ to whichever association this install uses. A no-op
     # when the host app configured no owner model.
     def owner=(record)
       association = self.class.owner_association
-      public_send(:"#{association}=", record) if association
+      return unless association
+      return public_send(:"#{association}=", record) if respond_to?(:"#{association}=")
+
+      self[:"#{association}_id"] = record&.id
     end
   end
 end