actionagent 1.6.2 → 1.6.3

data/app/models/action_agent/agent.rb

@@ -11,6 +11,14 @@ module ActionAgent
     has_many :agent_runs, dependent: :destroy
     has_many :evaluations, dependent: :destroy
     has_many :agent_memories, as: :memorable, dependent: :destroy
+    # Generations hang off AgentContext polymorphically, which is an
+    # implementation detail of how contexts are modelled — so without these a
+    # host that wants an agent's recorded history writes that join itself and is
+    # coupled to the shape. Deliberately no `dependent:` on the contexts: the
+    # association is added to read them, and destroying an agent has never taken
+    # its conversations with it. Making it do so is a separate call.
+    has_many :agent_contexts, as: :contextable
+    has_many :generations, through: :agent_contexts
 
     # Polymorphic rows (agent_memories, agent_contexts) store this string.
     # A host app that grew these tables under its own Agent constant keeps

data/app/models/action_agent/evaluation.rb

@@ -100,10 +100,16 @@ module ActionAgent
     # ActiveAgent::Evals::ScenarioParser output). Keys already in the suite keep their records, so
     # earlier runs' results still resolve to their scenario, and keep their
     # enabled flag unless the attributes set it (a paste cannot).
-    def replace_scenarios!(attributes)
+    # `on_removed:` decides what happens to a scenario the new attributes no
+    # longer name: `:destroy` drops it, `:disable` keeps the row with
+    # `enabled: false` so runs that scored it still resolve their results.
+    def replace_scenarios!(attributes, on_removed: :destroy)
+      raise ArgumentError, "on_removed must be :destroy or :disable" unless %i[destroy disable].include?(on_removed)
+
       transaction do
         keep = attributes.map { |attrs| attrs["key"] }
-        scenarios.where.not(key: keep).destroy_all
+        removed = scenarios.where.not(key: keep)
+        on_removed == :disable ? removed.update_all(enabled: false) : removed.destroy_all
 
         attributes.each_with_index do |attrs, index|
           scenario = scenarios.find_or_initialize_by(key: attrs["key"])

data/app/services/action_agent/agent_execution_service.rb

@@ -33,9 +33,13 @@ module ActionAgent
     # so a multi-gigabyte log named .csv costs a fixed slice of memory rather
     # than its whole size. Four bytes per character is UTF-8's worst case.
     ATTACHMENT_TEXT_BYTE_LIMIT = ATTACHMENT_TEXT_LIMIT * 4
+    # What one prompt-span attribute stores. The value is a preview for reading,
+    # so it is clipped; a size that has to stay exact travels as its own
+    # `*.tokens` attribute instead.
+    PROMPT_SPAN_ATTRIBUTE_LIMIT = 6000
     # The prompt span records the transcript, not the data URIs; keep the
     # whole serialized list within the same budget as the other attributes.
-    PROMPT_SPAN_MESSAGE_LIMIT = 6000
+    PROMPT_SPAN_MESSAGE_LIMIT = PROMPT_SPAN_ATTRIBUTE_LIMIT
     # Prior turns sent with a pinned conversation: the most recent ones,
     # trimmed oldest-first to a character budget.
     HISTORY_TURN_LIMIT = 40
@@ -164,11 +168,19 @@ module ActionAgent
     def record_prompt_span(root_span)
       span = root_span.add_span("agent.prompt", span_type: :prompt)
       if composed_instructions.present?
-        span.set_attribute("prompt.input.instructions", composed_instructions.to_s.byteslice(0, 6000).to_s.scrub)
-      end
-      if tool_schemas.present?
-        span.set_attribute("prompt.input.tools", tool_schemas.to_json.byteslice(0, 6000).to_s.scrub)
+        instructions = composed_instructions.to_s
+        span.set_attribute("prompt.input.instructions", instructions.byteslice(0, PROMPT_SPAN_ATTRIBUTE_LIMIT).to_s.scrub)
+        span.set_attribute("prompt.input.instructions.tokens", estimated_tokens(instructions))
       end
+      # MCP and toolbox schemas are attributed separately so the meter can name
+      # which half fills the window, and each carries its size. The content
+      # attributes are truncated previews for reading: sizing the context from
+      # one understates it by whatever the clip dropped, which for a twelve-tool
+      # agent is most of the schema.
+      record_tool_schema_attributes(span)
+      full_transcript_json = prompt_turn[:transcript].map do |message|
+        { role: message[:role], content: message[:content].to_s }
+      end.to_json
       transcript = prompt_turn[:transcript].map do |message|
         { role: message[:role], content: message[:content].to_s.byteslice(0, 4000).to_s.scrub }
       end
@@ -181,6 +193,13 @@ module ActionAgent
       end
       span.set_attribute("prompt.input.messages", serialized)
       span.set_attribute("messages.count", transcript.size)
+      # The stored attribute is the tail of the history that fit, so its size is
+      # not the transcript's. The meter apportions the provider's prompt_tokens
+      # across the segments it can size, and a transcript missing from that set
+      # is not merely imprecise: the segments that remain are scaled up to cover
+      # it, so a long conversation reads as an enormous system prompt. Measured
+      # over the full turn, before either the per-message clip or the trim.
+      span.set_attribute("prompt.input.messages.tokens", estimated_tokens(full_transcript_json))
       span.finish
     rescue StandardError => e
       Rails.logger.warn("[AgentExecutionService] prompt span failed: #{e.message}")
@@ -385,6 +404,30 @@ module ActionAgent
       @mcp_dispatcher ||= MCPToolDispatcher.new(@agent_record)
     end
 
+    # Splits the offered schemas the way `tool_schemas` assembles them, so the
+    # span reports what the model was actually sent: nothing for a mock run, and
+    # one MCP round trip rather than a second one for telemetry.
+    def record_tool_schema_attributes(span)
+      mcp_definitions, toolbox_definitions = tool_schema_halves
+      {
+        "prompt.input.tools" => toolbox_definitions,
+        "prompt.input.mcp_tools" => mcp_definitions
+      }.each do |key, definitions|
+        next if definitions.blank?
+
+        json = definitions.to_json
+        span.set_attribute(key, json.byteslice(0, PROMPT_SPAN_ATTRIBUTE_LIMIT).to_s.scrub)
+        span.set_attribute("#{key}.tokens", estimated_tokens(json))
+      end
+    end
+
+    # ~4 chars/token, the same approximation the context meter applies to content
+    # it sizes itself. Taken before truncation, so the meter reads the whole
+    # schema rather than the preview the attribute stores.
+    def estimated_tokens(text)
+      (text.length / 4.0).round
+    end
+
     def call_agent(slug:, message:)
       depth = Thread.current[:agent_call_depth].to_i
       return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
@@ -562,13 +605,21 @@ module ActionAgent
     # server-side implementations (none for mock runs — the mock provider
     # doesn't do tool calling).
     def tool_schemas
-      return [] if provider == :mock
-
-      # The agent's own MCP servers describe their tools; the toolbox describes
-      # the rest. Without the first half a tool the agent declares is never
-      # offered to the model, which then answers from memory instead of calling
-      # it.
-      mcp_dispatcher.tool_definitions + AgentToolbox.definitions_for(@agent_record.tools)
+      mcp_definitions, toolbox_definitions = tool_schema_halves
+      mcp_definitions + toolbox_definitions
+    end
+
+    # The agent's own MCP servers describe their tools; the toolbox describes the
+    # rest. Without the first half a tool the agent declares is never offered to
+    # the model, which then answers from memory instead of calling it. Memoized
+    # because listing a server's tools is a request to that server.
+    def tool_schema_halves
+      @tool_schema_halves ||=
+        if provider == :mock
+          [ [], [] ]
+        else
+          [ mcp_dispatcher.tool_definitions, AgentToolbox.definitions_for(@agent_record.tools) ]
+        end
     end
 
     # Persists the tool interaction stream to the solid_agent conversation

data/app/services/action_agent/evaluation_runner_service.rb

@@ -149,9 +149,7 @@ module ActionAgent
     end
 
     def sample_generations(model: nil)
-      scope = AgentGeneration
-        .joins(:agent_context)
-        .where(AgentContext.table_name => { contextable: @evaluation.agent })
+      scope = @evaluation.agent.generations
       scope = scope.where(model: model) if model
       scope.order(created_at: :desc).limit(@evaluation.sample_size).to_a
     end

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -64,7 +64,16 @@ module ActionAgent
       report = if adapter
         raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
 
-        adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
+        # The default path meters each replay itself (see #replay). An adapter
+        # runs the agent in the host's own runtime, out of reach of that call,
+        # so the execution is counted here as each result lands — one per
+        # scenario x model, the same unit. Otherwise a host that adapts the
+        # replay is silently unmetered unless it remembers to meter itself.
+        metered = lambda do |result|
+          ActionAgent.record_usage(owner, :execution)
+          on_result.call(result)
+        end
+        adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: metered)
       else
         ensure_judge_defined_kpis! if @evaluation.judge_defined?
         default_report(tasks, specs, on_result)
@@ -122,6 +131,12 @@ module ActionAgent
     def model_specs
       names = Array(@selection[:models]).presence || @evaluation.compare_models
       specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
+      # parse_all resolves a bare name against `providers:` but passes through a
+      # `provider/model` whose provider is not in that list, so the run would
+      # otherwise reach the replay with a provider nothing can serve.
+      unsupported = specs.map(&:provider).uniq - (Agent::PROVIDERS + %w[mock])
+      raise ArgumentError, "unsupported model provider: #{unsupported.to_sentence}" if unsupported.any?
+
       return specs if specs.any?
 
       [ Evals::ModelSpec.new(label: @evaluation.agent.model, provider: @evaluation.agent.provider, model: @evaluation.agent.model) ]

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.6.2"
+  VERSION = "1.6.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-17 00:00:00.000000000 Z
+date: 2026-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent