actionagent 1.2.2 → 1.5.0

data/app/services/action_agent/agent_execution_service.rb

@@ -23,6 +23,24 @@ module ActionAgent
 
     SERVICE_NAME = "activeagents-platform"
 
+    # Images and PDFs above this size are described rather than sent —
+    # a data URI of that size is most of a context window by itself.
+    ATTACHMENT_DATA_LIMIT = 8.megabytes
+    # Inlined text attachments are cut here: enough for a CSV or a config
+    # file, not enough for a log dump to crowd out the conversation.
+    ATTACHMENT_TEXT_LIMIT = 20_000
+    # ...and only this many bytes are ever read to produce those characters,
+    # so a multi-gigabyte log named .csv costs a fixed slice of memory rather
+    # than its whole size. Four bytes per character is UTF-8's worst case.
+    ATTACHMENT_TEXT_BYTE_LIMIT = ATTACHMENT_TEXT_LIMIT * 4
+    # The prompt span records the transcript, not the data URIs; keep the
+    # whole serialized list within the same budget as the other attributes.
+    PROMPT_SPAN_MESSAGE_LIMIT = 6000
+    # Prior turns sent with a pinned conversation: the most recent ones,
+    # trimmed oldest-first to a character budget.
+    HISTORY_TURN_LIMIT = 40
+    HISTORY_CHAR_BUDGET = 60_000
+
     def self.call(agent_record, run)
       new(agent_record, run).call
     end
@@ -59,6 +77,7 @@ module ActionAgent
     end
 
     def call
+      @agent_record.ensure_executable!
       root_span = @root_span = build_root_span
       record_prompt_span(root_span)
       llm_span = root_span.add_span(
@@ -124,7 +143,9 @@ module ActionAgent
     # The outbound prompt as a span, in the SDK's attribute shape — gives the
     # Traces UI its System/User conversation rows and lets the context-pressure
     # meter attribute instructions and tool schemas instead of lumping the
-    # whole input into "messages".
+    # whole input into "messages". Messages are the text-only transcript:
+    # the same list the provider gets, with data URIs replaced by
+    # "[image: sales_chart.png]" placeholders.
     def record_prompt_span(root_span)
       span = root_span.add_span("agent.prompt", span_type: :prompt)
       if composed_instructions.present?
@@ -133,16 +154,69 @@ module ActionAgent
       if tool_schemas.present?
         span.set_attribute("prompt.input.tools", tool_schemas.to_json.byteslice(0, 6000).to_s.scrub)
       end
-      span.set_attribute(
-        "prompt.input.messages",
-        [ { role: "user", content: @run.input_prompt.to_s.byteslice(0, 4000).to_s.scrub } ].to_json
-      )
-      span.set_attribute("messages.count", 1)
+      transcript = prompt_turn[:transcript].map do |message|
+        { role: message[:role], content: message[:content].to_s.byteslice(0, 4000).to_s.scrub }
+      end
+      # A replayed conversation is re-sent every turn, so the span keeps the
+      # most recent messages that fit rather than the whole history again.
+      serialized = transcript.to_json
+      while serialized.bytesize > PROMPT_SPAN_MESSAGE_LIMIT && transcript.size > 1
+        transcript.shift
+        serialized = transcript.to_json
+      end
+      span.set_attribute("prompt.input.messages", serialized)
+      span.set_attribute("messages.count", transcript.size)
       span.finish
     rescue StandardError => e
       Rails.logger.warn("[AgentExecutionService] prompt span failed: #{e.message}")
     end
 
+    # The list handed to prompt(messages:): the pinned conversation's prior
+    # turns, then the new user turn carrying the run's attachments — images
+    # and PDFs as data URIs in the provider-neutral {text:, image:} /
+    # {document:} shorthand, text files inlined, anything else described.
+    # Memoized, so the provider and the prompt span see one list.
+    def prompt_messages
+      prompt_turn[:messages]
+    end
+
+    # Persists the tool calls this service executed when the provider's
+    # response carries no tool-role messages to persist them from (the
+    # OpenAI Responses API). With such messages present the usual path
+    # — solid_agent's, then #persist_tool_messages — already writes the
+    # rows, keyed by tool_call_id, and this is a no-op.
+    def persist_tool_invocations(context, response)
+      return unless context.respond_to?(:add_tool_message)
+      return if @tool_invocations.empty?
+      return if Array(response.respond_to?(:messages) ? response.messages : nil).any? do |message|
+        message.respond_to?(:role) && message.role.to_s == "tool"
+      end
+
+      @tool_invocations.each do |invocation|
+        context.add_tool_message(
+          tool_call_id: nil,
+          tool_name: invocation[:name],
+          result: invocation[:result],
+          arguments: invocation[:arguments],
+          duration_ms: invocation[:duration_ms]
+        )
+      end
+    rescue StandardError => e
+      Rails.logger.error("[AgentExecutionService] Failed to persist tool invocations: #{e.message}")
+    end
+
+    # The person's own words for this turn — what the persisted user
+    # message says, without the file bodies inlined for the model.
+    def user_text
+      @run.input_prompt.to_s.presence || (attachment_records.any? ? "(see attached files)" : "")
+    end
+
+    # What was attached, as stored on the persisted user message so the
+    # conversation shows the files afterwards.
+    def attachment_manifest
+      @attachment_manifest ||= @run.attachment_manifest
+    end
+
     # Per-run provider/model overrides (input_params) let callers replay the
     # same agent under a different model — the basis of evaluation comparison
     # runs. Absent overrides, the agent's own configuration applies.
@@ -233,7 +307,9 @@ module ActionAgent
         when "call_agent"
           call_agent(slug: kwargs[:slug], message: kwargs[:message])
         else
-          AgentToolbox.call(name, **kwargs)
+          # A tool one of the agent's own MCP servers serves is called there;
+          # AgentToolbox answers the rest.
+          mcp_dispatcher.call(name, kwargs) || AgentToolbox.call(name, **kwargs)
         end
       rescue StandardError => e
         Rails.logger.warn("[AgentExecutionService] Tool #{name} failed: #{e.class} - #{e.message}")
@@ -262,6 +338,7 @@ module ActionAgent
       @tool_invocations << {
         name: name.to_s,
         arguments: kwargs,
+        result: result,
         duration_ms: duration_ms,
         error: errored
       }
@@ -279,6 +356,12 @@ module ActionAgent
     # Executes another agent of the same account synchronously and returns
     # its reply, so agents can delegate to each other as a tool call. The
     # sub-run is a real AgentRun with its own trace.
+    # One dispatcher per run, so every tool call shares the MCP sessions the
+    # first call opens.
+    def mcp_dispatcher
+      @mcp_dispatcher ||= MCPToolDispatcher.new(@agent_record)
+    end
+
     def call_agent(slug:, message:)
       depth = Thread.current[:agent_call_depth].to_i
       return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
@@ -341,7 +424,10 @@ module ActionAgent
       model_options.merge!(owner_provider_options(effective_provider))
       klass_name = agent_class_name
       agent_record = @agent_record
-      input = @run.input_prompt
+      # Pinned, or the default stream resolved here rather than left to
+      # solid_agent's unordered find_or_create_by!, which cannot tell the
+      # agent's original conversation from a later one on the same triple.
+      pinned = conversation_context
       instructions = composed_instructions
       action = action_name
       run_trace_id = trace_id
@@ -395,19 +481,52 @@ module ActionAgent
 
         # One method per invokable action (the default plus each named action
         # prompt) — solid_agent keys the persisted context by action_name, so
-        # each action gets its own interaction stream.
+        # each action gets its own interaction stream. A run pinned to a
+        # conversation continues that context instead.
         define_method action do
           # Thread the run's telemetry trace_id through prompt_options so
           # SolidAgent's provenance (and AgentContext#record_generation_with_
           # provenance!) can correlate the persisted generation with its trace.
           prompt_options[:trace_id] = run_trace_id
-          load_context(contextable: agent_record)
+          if pinned
+            load_context(context_id: pinned.id)
+          else
+            load_context(contextable: agent_record)
+          end
 
-          options = { message: input }
+          options = { messages: service.prompt_messages }
           options[:instructions] = instructions if instructions.present?
           options[:tools] = tool_definitions if tool_definitions.present?
           prompt(**options)
         end
+
+        # solid_agent's after_prompt callback persists the last prompt
+        # message's content: string — so a turn that carries files (a
+        # {text:, image:} hash) would never be written, and the history
+        # replayed from the pinned context is not this run's to persist.
+        # Instead: exactly one user message per run, through the agent-level
+        # add_user_message that stamps provenance (its trace_id is how the
+        # run detail API finds the run's slice of the conversation), with
+        # the attachment manifest alongside.
+        define_method(:persist_prompt_to_context) do
+          text = service.user_text
+          return unless context && text.present?
+
+          add_user_message(text, attachments: service.attachment_manifest)
+        end
+        private :persist_prompt_to_context
+
+        # solid_agent persists the tool exchange from the response's
+        # tool-role messages. The Responses API carries function calls as
+        # items rather than messages, so that list is empty and the
+        # exchange — a render_ui call is the reply — would vanish from the
+        # conversation. The service saw every call go by; fall back to its
+        # own records, here so the rows land before the assistant turn.
+        define_method(:persist_tool_messages_to_context) do
+          super()
+          service.persist_tool_invocations(context, generation_response)
+        end
+        private :persist_tool_messages_to_context
       end
 
       agent_class.public_send(action).generate_now
@@ -419,7 +538,11 @@ module ActionAgent
     def tool_schemas
       return [] if provider == :mock
 
-      AgentToolbox.definitions_for(@agent_record.tools)
+      # The agent's own MCP servers describe their tools; the toolbox describes
+      # the rest. Without the first half a tool the agent declares is never
+      # offered to the model, which then answers from memory instead of calling
+      # it.
+      mcp_dispatcher.tool_definitions + AgentToolbox.definitions_for(@agent_record.tools)
     end
 
     # Persists the tool interaction stream to the solid_agent conversation
@@ -530,10 +653,165 @@ module ActionAgent
       @trace_id ||= @run.trace_id.presence || SecureRandom.hex(16)
     end
 
-    # The solid_agent conversation context this execution persisted into
-    # (one per agent + action on this platform).
+    # The solid_agent conversation context this execution persisted into:
+    # the pinned one when the run continues a conversation, else the
+    # agent + action's default stream.
     def conversation_context
-      AgentContext.find_by(contextable: @agent_record, agent_name: agent_class_name, action_name: action_name)
+      pinned_context || default_stream_context
+    end
+
+    # The agent + action's default stream: the oldest context on the triple
+    # solid_agent keys by, so a second conversation the dashboard started for
+    # the same action cannot become the row an unpinned run appends to.
+    def default_stream_context
+      AgentContext.where(contextable: @agent_record, agent_name: agent_class_name, action_name: action_name)
+        .order(:id).first
+    end
+
+    # The conversation the run was pinned to (input_params context_id). A
+    # context belonging to another agent — or recorded under another action,
+    # whose instructions and stream are not this run's — is ignored rather
+    # than continued: the run falls back to the default stream as if nothing
+    # had been pinned, and reports the conversation it actually wrote to.
+    def pinned_context
+      return @pinned_context if defined?(@pinned_context)
+
+      id = run_params[:context_id]
+      @pinned_context =
+        if id.present?
+          # Matched on the action too, not just ownership: a run for another
+          # action would append to this conversation and rewrite the recorded
+          # instructions with its own. The agent_name is deliberately not part
+          # of it — renaming an agent changes that string, and the
+          # conversations it already has must stay pinnable.
+          AgentContext.find_by(id: id, contextable: @agent_record, action_name: action_name)
+        end
+    end
+
+    # The pinned conversation's prior turns as plain {role:, content:}
+    # messages — the conversation as the person saw it: tool rows and empty
+    # assistant rows (a turn that only carried a tool call) are skipped.
+    # The most recent turns, dropped oldest-first once the budget is spent.
+    def history_messages
+      context = pinned_context
+      return [] unless context
+
+      turns = context.messages.chronological
+        .where(role: %w[user assistant])
+        .where.not(content: [ nil, "" ])
+        .last(HISTORY_TURN_LIMIT)
+
+      budget = HISTORY_CHAR_BUDGET
+      kept = turns.reverse_each.with_object([]) do |message, collected|
+        content = message.content.to_s
+        break collected if content.length > budget
+
+        budget -= content.length
+        collected.unshift(role: message.role, content: content)
+      end
+
+      # Neither cut lands on a turn boundary, so the oldest survivor can be an
+      # assistant reply whose question was dropped. Anthropic rejects a
+      # conversation that opens on one, and every provider reads it as an
+      # answer to nothing.
+      kept.shift while kept.first && kept.first[:role] != "user"
+      kept
+    end
+
+    # Builds the message list and, in the same pass, its text-only
+    # transcript for the prompt span (data URIs are too big to trace).
+    #
+    # The first image or document rides on the user's text as a
+    # {text:, image:} / {text:, document:} message; each further one is a
+    # message of its own, since the shorthand carries one part per key.
+    def prompt_turn
+      @prompt_turn ||= begin
+        # dup: the inlined file bodies must not land on the run's own
+        # input_prompt through in-place mutation.
+        text = user_text.dup
+        media = []
+
+        attachment_records.each do |attachment|
+          blob = attachment.blob
+          filename = blob.filename.to_s
+          descriptor = "#{filename} (#{blob.content_type}, #{human_size(blob.byte_size)})"
+
+          # A file the storage service can no longer produce costs the file,
+          # not the run: every branch below degrades to the same descriptor
+          # the unsupported kinds get.
+          begin
+            case AgentRun.attachment_kind(blob.content_type, filename)
+            when "text"
+              body = text_prefix(blob)
+              suffix = body.bytesize < blob.byte_size ? "\n… (truncated)" : ""
+              text << "\n\n[Attached file: #{descriptor}]\n```\n#{body}#{suffix}\n```"
+            when "image", "document"
+              if blob.byte_size > ATTACHMENT_DATA_LIMIT
+                text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
+              else
+                key = blob.content_type.to_s.start_with?("image/") ? :image : :document
+                media << { key => data_uri(blob), label: "[#{key}: #{filename}]" }
+              end
+            else
+              text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
+            end
+          rescue StandardError => e
+            Rails.logger.warn("[AgentExecutionService] attachment #{filename} unreadable: #{e.message}")
+            text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
+          end
+        end
+
+        first, *rest = media
+        history = history_messages
+        # No prompt and no files sends no turn at all, which leaves the
+        # gem's template fallback in charge exactly as before.
+        turn =
+          if first
+            { role: "user", text: text }.merge(first.except(:label))
+          elsif text.present?
+            { role: "user", content: text }
+          end
+        {
+          messages: history + [ turn ].compact + rest.map { |item| { role: "user" }.merge(item.except(:label)) },
+          transcript: history +
+            [ turn && { role: "user", content: [ text, first&.dig(:label) ].compact.join("\n") } ].compact +
+            rest.map { |item| { role: "user", content: item[:label] } }
+        }
+      end
+    end
+
+    def attachment_records
+      @attachment_records ||= AgentRun.attachments_available? ? @run.attachments_attachments.includes(:blob).order(:id).to_a : []
+    end
+
+    def data_uri(blob)
+      "data:#{blob.content_type};base64,#{Base64.strict_encode64(blob.download)}"
+    end
+
+    # The head of a text attachment, reading a bounded number of bytes: only
+    # ATTACHMENT_TEXT_LIMIT characters are ever sent, so a huge file must not
+    # be materialised whole to produce them. The byte prefix can split a
+    # multibyte character, which scrub removes.
+    def text_prefix(blob)
+      bytes =
+        if blob.byte_size <= ATTACHMENT_TEXT_BYTE_LIMIT
+          blob.download
+        elsif blob.service.respond_to?(:download_chunk)
+          blob.service.download_chunk(blob.key, 0...ATTACHMENT_TEXT_BYTE_LIMIT)
+        else
+          buffer = +""
+          blob.download do |chunk|
+            buffer << chunk
+            break if buffer.bytesize >= ATTACHMENT_TEXT_BYTE_LIMIT
+          end
+          buffer
+        end
+
+      bytes.to_s.dup.force_encoding(Encoding::UTF_8).scrub[0, ATTACHMENT_TEXT_LIMIT]
+    end
+
+    def human_size(bytes)
+      ActiveSupport::NumberHelper.number_to_human_size(bytes, precision: 2)
     end
 
     # The agent's owner under the configured mode; nil when the install
@@ -563,7 +841,7 @@ module ActionAgent
       tenant = ActionAgent.tenant_for(owner)
       return if trace_model.for_account(tenant).exists?(trace_id: root_span.trace_id)
 
-      trace_model.create_from_payload(payload, sdk_info, account: tenant)
+      trace_model.create_from_payload(payload, sdk_info, account: tenant, agent: @agent_record)
     rescue StandardError => e
       Rails.logger.error("[AgentExecutionService] Failed to record trace #{root_span.trace_id}: #{e.class} - #{e.message}")
       nil

data/app/services/action_agent/agent_registrar.rb

@@ -10,9 +10,10 @@ module ActionAgent
   # metrics can hang off.
   #
   # Identity is (account, service_name, agent_class, agent_action) — one agent
-  # per action, not per class. Clara.respond (admin assistant, every MCP tool,
-  # ~$0.03/run) and Clara.title (no tools, temperature 0.2, ~$0.0004/run) are
-  # different agents that share a class name because one app method spawns both.
+  # per action, not per class. Assistant.respond (admin assistant, every MCP
+  # tool, ~$0.03/run) and Assistant.title (no tools, temperature 0.2,
+  # ~$0.0004/run) are different agents that share a class name because one app
+  # method spawns both.
   # Collapsing them would blend a $0.03 agent with a $0.0004 one into a single
   # meaningless cost-per-run.
   #
@@ -60,7 +61,7 @@ module ActionAgent
       @trace.agent_action.presence
     end
 
-    # Name carries the action so the two Claras are distinguishable anywhere a
+    # Name carries the action so the two agents are distinguishable anywhere a
     # bare agent name is shown.
     def display_name
       action_name ? "#{agent_class}.#{action_name}" : agent_class
@@ -138,8 +139,8 @@ module ActionAgent
 
     # Config we can only learn by watching: the model actually used, the
     # instructions actually sent (when content capture is on), and the tools
-    # actually called — which is how Clara.respond acquires a tool list while
-    # Clara.title correctly stays empty.
+    # actually called — which is how Assistant.respond acquires a tool list while
+    # Assistant.title correctly stays empty.
     def llm_attribute(key)
       spans.filter_map { |span| span.dig("attributes", key).presence }.first
     end

data/app/services/action_agent/agent_toolbox.rb

@@ -46,7 +46,7 @@ module ActionAgent
           }
         }
       ],
-      # A real browser via a Playwright MCP server (PlaywrightMcpClient).
+      # A real browser via a Playwright MCP server (PlaywrightMCPClient).
       # Stateful: navigate changes what snapshot/click see, so these bypass
       # the toolbox result cache.
       "playwright_mcp" => [
@@ -121,6 +121,33 @@ module ActionAgent
           }
         }
       ],
+      # Generative UI. The call is the output: the runner renders the
+      # blocks straight from the persisted tool arguments, so the
+      # implementation only has to acknowledge them.
+      "ui" => [
+        {
+          name: "render_ui",
+          description: "Render interactive UI for the user instead of (or alongside) prose. Pass an array of blocks. Block types and fields: " \
+            "card {title, body (markdown), image_url?, footer?}; stat {label, value, delta?, tone? (positive|negative|neutral)}; " \
+            "stats {items: [stat...]}; table {columns: [string], rows: [[cell...]]}; " \
+            "chart {chart: bar|line|area|pie, title?, x (key), series: [key...], data: [{...}]}; " \
+            "list {title?, items: [string], ordered?}; progress {label, value (0-100)}; " \
+            "form {title?, submit? (button label), fields: [{name, label, type (text|textarea|number|select|checkbox), options?: [string], placeholder?, required?}]}; " \
+            "choices {prompt?, options: [string]}; image {url, alt?, caption?}; callout {tone (info|success|warning|danger), title?, body}; " \
+            "code {language?, code}. Blocks render top to bottom.",
+          parameters: {
+            type: "object",
+            properties: {
+              blocks: {
+                type: "array",
+                description: "UI blocks to render, in order",
+                items: { type: "object", properties: { type: { type: "string" } }, required: [ "type" ] }
+              }
+            },
+            required: [ "blocks" ]
+          }
+        }
+      ],
       # Memory tools mirror solid_agent's HasMemory contract. They are NOT in
       # FUNCTIONS below — execution is subject-bound, so AgentExecutionService
       # routes them to the run's AgentMemory instead of this module.
@@ -160,11 +187,14 @@ module ActionAgent
       "browse_page" => :browse_page,
       "browser_navigate" => :browser_navigate,
       "browser_snapshot" => :browser_snapshot,
-      "browser_click" => :browser_click
+      "browser_click" => :browser_click,
+      "render_ui" => :render_ui
     }.freeze
 
-    # Stateful tools whose results must never be replayed from cache.
-    UNCACHED_FUNCTIONS = %w[browser_navigate browser_snapshot browser_click].freeze
+    # Stateful tools whose results must never be replayed from cache — and
+    # render_ui, whose result is the call itself, so there is nothing to
+    # replay.
+    UNCACHED_FUNCTIONS = %w[browser_navigate browser_snapshot browser_click render_ui].freeze
 
     # Hosts browse_page may fetch — the platform's own trusted docs.
     BROWSE_ALLOWED_HOSTS = %w[docs.activeagents.ai].freeze
@@ -258,6 +288,18 @@ module ActionAgent
         { error: e.message }
       end
 
+      # No side effects: the blocks are rendered by the runner from the tool
+      # call's arguments. Malformed blocks are reported back so the model can
+      # fix its call rather than shipping UI the runner would drop.
+      def render_ui(blocks:)
+        valid = blocks.is_a?(Array) && blocks.all? do |block|
+          block.respond_to?(:key?) && (block[:type] || block["type"]).is_a?(String)
+        end
+        return { error: "blocks must be an array of objects, each with a string type" } unless valid
+
+        { rendered: true, blocks: blocks.size }
+      end
+
       # Trusted-docs browser: fetch_url restricted to BROWSE_ALLOWED_HOSTS,
       # with HTML reduced to readable text so small models aren't drowned in
       # markup. Accepts bare paths ("/docs/agents") against the docs host.
@@ -288,17 +330,17 @@ module ActionAgent
       SNAPSHOT_LINK = /\[Snapshot\]\(([^)]+)\)/
 
       def playwright_mcp(tool, arguments, retried: false)
-        result = PlaywrightMcpClient.instance.call_tool(tool, arguments)
+        result = PlaywrightMCPClient.instance.call_tool(tool, arguments)
         text = inline_snapshot(result[:text].to_s)
         if text.length > PLAYWRIGHT_RESULT_LIMIT
           text = "#{text[0, PLAYWRIGHT_RESULT_LIMIT]}\n…(truncated, #{text.length} chars total)"
         end
         result[:is_error] ? { error: text.presence || "browser tool failed" } : { text: text }
-      rescue PlaywrightMcpClient::Error => e
+      rescue PlaywrightMCPClient::Error => e
         # One fresh-session retry: the first call after a server (re)start can
         # race the browser launch.
         unless retried
-          PlaywrightMcpClient.reset!
+          PlaywrightMCPClient.reset!
           return playwright_mcp(tool, arguments, retried: true)
         end
         { error: e.message }