activeagent 0.6.3 → 1.0.0

data/lib/active_agent/providers/concerns/instrumentation.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Providers
+    # Builds instrumentation event payloads for ActiveSupport::Notifications.
+    #
+    # Extracts request parameters and response metadata for monitoring, debugging,
+    # and APM integration (New Relic, DataDog, etc.).
+    #
+    # == Event Payloads
+    #
+    # Top-Level Events (overall request lifecycle):
+    #
+    # prompt.active_agent::
+    #   Initial: `{ model:, temperature:, max_tokens:, message_count:, has_tools:, stream: }`
+    #   Final: `{ usage: { input_tokens:, output_tokens:, total_tokens: }, finish_reason:, response_model:, response_id: }`
+    #   Note: Usage is cumulative across all API calls in multi-turn conversations
+    #
+    # embed.active_agent::
+    #   Initial: `{ model:, input_size:, encoding_format:, dimensions: }`
+    #   Final: `{ usage: { input_tokens:, total_tokens: }, embedding_count:, response_model:, response_id: }`
+    #
+    # Provider-Level Events (per API call):
+    #
+    # prompt.provider.active_agent::
+    #   Initial: `{ model:, temperature:, max_tokens:, message_count:, has_tools:, stream: }`
+    #   Final: `{ usage: { input_tokens:, output_tokens:, total_tokens: }, finish_reason:, response_model:, response_id: }`
+    #   Note: Usage is per individual API call
+    #
+    # embed.provider.active_agent::
+    #   Initial: `{ model:, input_size:, encoding_format:, dimensions: }`
+    #   Final: `{ usage: { input_tokens:, total_tokens: }, embedding_count:, response_model:, response_id: }`
+    module Instrumentation
+      extend ActiveSupport::Concern
+
+      # Builds and merges payload data for prompt instrumentation events.
+      #
+      # Populates both request parameters and response metadata for top-level and
+      # provider-level events. Usage data (tokens) is CRITICAL for APM cost tracking
+      # and performance monitoring.
+      #
+      # @param payload [Hash] instrumentation payload to merge into
+      # @param request [Request] request object with parameters
+      # @param response [Common::PromptResponse] completed response with normalized data
+      # @return [void]
+      def instrumentation_prompt_payload(payload, request, response)
+        # message_count: prefer the request/input messages (pre-call), fall back to
+        # response messages only if the request doesn't expose messages. New Relic
+        # expects parameters[:messages] to be the request messages and computes
+        # total message counts by adding response choices to that count.
+        message_count = safe_access(request, :messages)&.size
+        message_count = safe_access(response, :messages)&.size if message_count.nil?
+
+        payload.merge!(trace_id: trace_id, message_count: message_count || 0, stream: !!safe_access(request, :stream))
+
+        # Common parameters: prefer response-normalized values, then request
+        payload[:model]       = safe_access(response, :model) || safe_access(request, :model)
+        payload[:temperature] = safe_access(request, :temperature)
+        payload[:max_tokens]  = safe_access(request, :max_tokens)
+        payload[:top_p]       = safe_access(request, :top_p)
+
+        # Tools / instructions
+        if (tools_val = safe_access(request, :tools))
+          payload[:has_tools]  = tools_val.respond_to?(:present?) ? tools_val.present? : !!tools_val
+          payload[:tool_count] = tools_val&.size || 0
+        end
+
+        if (instr_val = safe_access(request, :instructions))
+          payload[:has_instructions] = instr_val.respond_to?(:present?) ? instr_val.present? : !!instr_val
+        end
+
+        # Usage (normalized)
+        if response.usage
+          usage = response.usage
+          payload[:usage] = {
+            input_tokens:  usage.input_tokens,
+            output_tokens: usage.output_tokens,
+            total_tokens:  usage.total_tokens
+          }
+
+          payload[:usage][:cached_tokens]         = usage.cached_tokens         if usage.cached_tokens
+          payload[:usage][:cache_creation_tokens] = usage.cache_creation_tokens if usage.cache_creation_tokens
+          payload[:usage][:reasoning_tokens]      = usage.reasoning_tokens      if usage.reasoning_tokens
+          payload[:usage][:audio_tokens]          = usage.audio_tokens          if usage.audio_tokens
+        end
+
+        # Response metadata
+        payload[:finish_reason]  = safe_access(response, :finish_reason) || response.finish_reason
+        payload[:response_model] = safe_access(response, :model)         || response.model
+        payload[:response_id]    = safe_access(response, :id)            || response.id
+
+        # Build messages list: prefer request messages; if unavailable use prior
+        # response messages (all but the final generated message).
+        if (req_msgs = safe_access(request, :messages)).is_a?(Array)
+          payload[:messages] = req_msgs.map { |m| extract_message_hash(m, false) }
+        else
+          prior = safe_access(response, :messages)
+          prior = prior[0...-1] if prior.is_a?(Array) && prior.size > 1
+          if prior.is_a?(Array) && prior.any?
+            payload[:messages] = prior.map { |m| extract_message_hash(m, false) }
+          end
+        end
+
+        # Build a parameters hash that mirrors what New Relic's OpenAI
+        # instrumentation expects. This makes it easy for APM adapters to
+        # map our provider payload to their LLM event constructors.
+        parameters = {}
+        parameters[:model]       = payload[:model]       if payload[:model]
+        parameters[:max_tokens]  = payload[:max_tokens]  if payload[:max_tokens]
+        parameters[:temperature] = payload[:temperature] if payload[:temperature]
+        parameters[:top_p]       = payload[:top_p]       if payload[:top_p]
+        parameters[:stream]      = payload[:stream]
+        parameters[:messages]    = payload[:messages]    if payload[:messages]
+
+        # Include tools/instructions where available — New Relic ignores unknown keys,
+        # but having them here makes the parameter shape closer to OpenAI's.
+        parameters[:tools]        = begin request.tools        rescue nil end if begin request.tools        rescue nil end
+        parameters[:instructions] = begin request.instructions rescue nil end if begin request.instructions rescue nil end
+
+        payload[:parameters] = parameters
+
+        # Attach raw response (provider-specific) so downstream APM integrations
+        # can inspect the provider response if needed. Use the normalized raw_response
+        # available on the Common::Response when possible.
+        begin
+          payload[:response_raw] = response.raw_response if response.respond_to?(:raw_response) && response.raw_response
+        rescue StandardError
+          # ignore
+        end
+      end
+
+      private
+
+      # Safely attempt to call a method or lookup a key on an object. We avoid
+      # probing with `respond_to?` to prevent ActiveModel attribute casting side
+      # effects; instead we attempt the call and rescue failures.
+      def safe_access(obj, name)
+        return nil if obj.nil?
+
+        begin
+          return obj.public_send(name)
+        rescue StandardError
+        end
+
+        begin
+          return obj[name]
+        rescue StandardError
+        end
+
+        begin
+          return obj[name.to_s]
+        rescue StandardError
+        end
+
+        nil
+      end
+
+      # NOTE: message access is handled via `safe_access(obj, :messages)` to
+      # avoid duplicating guarded lookup logic.
+
+      # Extract a simple hash from a provider message object or hash-like value.
+      def extract_message_hash(msg, is_response = false)
+        role = begin
+          if msg.respond_to?(:[])
+            begin msg[:role] rescue (begin msg["role"] rescue nil end) end
+          elsif msg.respond_to?(:role)
+            msg.role
+          elsif msg.respond_to?(:type)
+            msg.type
+          end
+        rescue StandardError
+          begin msg.role rescue msg.type rescue nil end
+        end
+
+        content = begin
+          if msg.respond_to?(:[])
+            begin msg[:content] rescue (begin msg["content"] rescue nil end) end
+          elsif msg.respond_to?(:content)
+            msg.content
+          elsif msg.respond_to?(:text)
+            msg.text
+          elsif msg.respond_to?(:to_h)
+            begin msg.to_h[:content] rescue (begin msg.to_h["content"] rescue nil end) end
+          elsif msg.respond_to?(:to_s)
+            msg.to_s
+          end
+        rescue StandardError
+          begin msg.to_s rescue nil end
+        end
+
+        { role: role, content: content, is_response: is_response }
+      end
+
+      # Builds and merges payload data for embed instrumentation events.
+      #
+      # Embeddings typically only report input tokens (no output tokens).
+      #
+      # @param payload [Hash] instrumentation payload to merge into
+      # @param request [Request] request object with parameters
+      # @param response [Common::EmbedResponse] completed response with normalized data
+      # @return [void]
+      def instrumentation_embed_payload(payload, request, response)
+        # Add request parameters
+        payload[:trace_id] = trace_id
+        payload[:model] = request.model if request.respond_to?(:model)
+
+        # Add input size if available
+        if request.respond_to?(:input)
+          begin
+            input = request.input
+            if input.is_a?(String)
+              payload[:input_size] = 1
+            elsif input.is_a?(Array)
+              payload[:input_size] = input.size
+            end
+          rescue # OpenAI throws errors this for some reason when you try to look at the input.
+            payload[:input_size] = request[:input].size
+          end
+        end
+
+        # Expose embedding input content similarly to message content.
+        # Use guarded access to avoid provider-specific errors.
+        begin
+          if (emb_input = safe_access(request, :input))
+            # Keep the raw input (string or array) in the payload so APM adapters
+            # can inspect it. This matches how we include message content.
+            payload[:input] = emb_input
+          end
+        rescue StandardError
+          # ignore
+        end
+
+        # Add encoding format if available (OpenAI)
+        payload[:encoding_format] = request.encoding_format if request.respond_to?(:encoding_format)
+
+        # Add dimensions if available (OpenAI)
+        payload[:dimensions] = request.dimensions if request.respond_to?(:dimensions)
+
+        # Add response data
+        payload[:embedding_count] = response.data&.size || 0
+
+        # Add usage data if available (CRITICAL for APM integration)
+        # Embeddings typically only have input tokens
+        if response.usage
+          payload[:usage] = {
+            input_tokens: response.usage.input_tokens,
+            total_tokens: response.usage.total_tokens
+          }
+        end
+
+        # Add response metadata directly from response object
+        payload[:response_model] = response.model
+        payload[:response_id]    = response.id
+
+        # Build a parameters hash for embeddings to match New Relic's shape.
+        emb_params = {}
+        emb_params[:model] = payload[:model] if payload[:model]
+        emb_params[:input] = payload[:input] if payload.key?(:input)
+        payload[:parameters] = emb_params unless emb_params.empty?
+      end
+    end
+  end
+end

data/lib/active_agent/providers/concerns/previewable.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Providers
+    # Generates markdown previews of prompts for debugging and inspection.
+    #
+    # Renders request parameters, instructions, messages, and tools in a
+    # human-readable format without executing the actual API call.
+    module Previewable
+      extend ActiveSupport::Concern
+
+      # Generates a markdown preview of the prompt request.
+      #
+      # @return [String] markdown-formatted preview
+      def preview_prompt
+        request = prepare_prompt_request
+
+        # @todo Validate Request
+        api_parameters = api_request_build(request, prompt_request_type)
+
+        render_markdown_preview(api_parameters)
+      end
+
+      private
+
+      # Renders markdown preview with YAML metadata, instructions, messages, and tools.
+      #
+      # Sections are separated by `---` dividers for readability.
+      #
+      # @param parameters [Hash]
+      # @return [String]
+      def render_markdown_preview(parameters)
+        sections = []
+
+        # Instructions section
+        if (instructions = parameters.delete(:instructions)).present?
+          sections << render_instructions_section(instructions)
+        end
+
+        # Messages section
+        if (messages = parameters.delete(:messages) || parameters.delete(:input)).present?
+          sections << render_messages_section(messages)
+        end
+
+        # Tools section
+        if (tools = parameters.delete(:tools)).present?
+          sections << render_tools_section(tools)
+        end
+
+        # Prepend YAML section with request details
+        sections = [ render_yaml_section(parameters), "---" ] + sections
+
+        sections.compact.join("\n")
+      end
+
+      # @param instructions [String]
+      # @return [String]
+      def render_instructions_section(instructions)
+        "## Instructions\n#{instructions}"
+      end
+
+      # Renders conversation messages with role labels.
+      #
+      # @param messages [Array<Hash>]
+      # @return [String]
+      def render_messages_section(messages)
+        return "" if messages.nil? || messages.empty?
+
+        content = +"## Messages\n\n"
+
+        Array(messages).each_with_index do |message, index|
+          content << render_single_message(message, index + 1)
+          content << "\n\n" unless index == messages.size - 1
+        end
+
+        content
+      end
+
+      # @param message [Hash]
+      # @param index [Integer] 1-based message number
+      # @return [String]
+      def render_single_message(message, index)
+        role    = (message.is_a?(Hash) && message[:role]) || "user"
+        content = extract_message_content(message)
+
+        "### Message #{index} (#{role.capitalize})\n#{content}"
+      end
+
+      # Renders available tools with descriptions and parameter schemas.
+      #
+      # @param tools [Array<Hash>]
+      # @return [String]
+      def render_tools_section(tools)
+        return "" if tools.nil? || tools.empty?
+
+        content = +"## Tools\n\n"
+
+        tools.each_with_index do |tool, index|
+          content << "### #{tool[:name] || "Tool #{index + 1}"}\n"
+          content << "**Description:** #{tool[:description] || 'No description'}\n\n"
+
+          if tool[:parameters]
+            content << "**Parameters:**\n```json\n#{JSON.pretty_generate(tool[:parameters])}\n```\n\n"
+          end
+        end
+
+        content.chomp
+      end
+
+      # Extracts text content from various message formats.
+      #
+      # Handles string messages, hash messages with :content key, and
+      # array content blocks (extracting only text-type blocks).
+      #
+      # @param message [Hash, String, nil]
+      # @return [String]
+      def extract_message_content(message)
+        return "" if message.nil?
+
+        case message
+        when String
+          message
+        when Hash
+          content = message[:content]
+          case content
+          when String
+            content
+          when Array
+            content
+              .select { |block| block.is_a?(Hash) && block[:type] == "text" }
+              .map { |block| block[:text] }
+              .join(" ")
+          else
+            content.to_s
+          end
+        else
+          message.to_s
+        end
+      end
+
+      # Renders request metadata as YAML.
+      #
+      # @param parameters [Hash]
+      # @return [String]
+      def render_yaml_section(parameters)
+        parameters.to_yaml.chomp
+      end
+    end
+  end
+end

data/lib/active_agent/providers/log_subscriber.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "active_support/log_subscriber"
+
+module ActiveAgent
+  module Providers
+    # Logs provider operations via ActiveSupport::Notifications events.
+    #
+    # Subscribes to instrumented provider events and formats them consistently.
+    # Customize by subclassing and attaching your subscriber, or adjust log levels.
+    #
+    # @example Custom log formatting
+    #   class MyLogSubscriber < ActiveAgent::Providers::LogSubscriber
+    #     def prompt(event)
+    #       info "🚀 #{event.payload[:provider_module]}: #{event.duration}ms"
+    #     end
+    #   end
+    #
+    #   ActiveAgent::Providers::LogSubscriber.detach_from :active_agent
+    #   MyLogSubscriber.attach_to :active_agent
+    class LogSubscriber < ActiveSupport::LogSubscriber
+      # self.namespace = "active_agent" # Rails 8.1
+
+      # Logs completed prompt with model, message count, token usage, and duration.
+      #
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def prompt(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+        model           = event.payload[:model]
+        message_count   = event.payload[:message_count]
+        stream          = event.payload[:stream]
+        usage           = event.payload[:usage]
+        finish_reason   = event.payload[:finish_reason]
+        duration        = event.duration.round(1)
+
+        debug do
+          parts = [ "[#{trace_id}]", "[ActiveAgent]", "[#{provider_module}]" ]
+          parts << "Prompt completed:"
+          parts << "model=#{model}" if model
+          parts << "messages=#{message_count}"
+          parts << "stream=#{stream}"
+
+          if usage
+            tokens = "tokens=#{usage[:input_tokens]}/#{usage[:output_tokens]}"
+            tokens += " (cached: #{usage[:cached_tokens]})" if usage[:cached_tokens]&.positive?
+            tokens += " (reasoning: #{usage[:reasoning_tokens]})" if usage[:reasoning_tokens]&.positive?
+            parts << tokens
+          end
+
+          parts << "finish=#{finish_reason}" if finish_reason
+          parts << "#{duration}ms"
+
+          parts.join(" ")
+        end
+      end
+      # event_log_level :prompt, :debug # Rails 8.1
+
+      # Logs completed embedding with model, input size, and token usage.
+      #
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def embed(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+        model           = event.payload[:model]
+        input_size      = event.payload[:input_size]
+        embedding_count = event.payload[:embedding_count]
+        usage           = event.payload[:usage]
+        duration        = event.duration.round(1)
+
+        debug do
+          parts = [ "[#{trace_id}]", "[ActiveAgent]", "[#{provider_module}]" ]
+          parts << "Embed completed:"
+          parts << "model=#{model}" if model
+          parts << "inputs=#{input_size}" if input_size
+          parts << "embeddings=#{embedding_count}" if embedding_count
+          parts << "tokens=#{usage[:input_tokens]}" if usage
+          parts << "#{duration}ms"
+
+          parts.join(" ")
+        end
+      end
+      # event_log_level :embed, :debug # Rails 8.1
+
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def stream_open(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+
+        debug do
+          "[#{trace_id}] [ActiveAgent] [#{provider_module}] Opening stream"
+        end
+      end
+      # event_log_level :stream_open, :debug # Rails 8.1
+
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def stream_close(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+
+        debug do
+          "[#{trace_id}] [ActiveAgent] [#{provider_module}] Closing stream"
+        end
+      end
+      # event_log_level :stream_close, :debug # Rails 8.1
+
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def tool_call(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+        tool_name       = event.payload[:tool_name]
+        duration        = event.duration.round(1)
+
+        debug do
+          "[#{trace_id}] [ActiveAgent] [#{provider_module}] Tool call: #{tool_name} (#{duration}ms)"
+        end
+      end
+      # event_log_level :tool_call, :debug # Rails 8.1
+
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def stream_chunk(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+        chunk_type      = event.payload[:chunk_type]
+
+        debug do
+          if chunk_type
+            "[#{trace_id}] [ActiveAgent] [#{provider_module}] Stream chunk: #{chunk_type}"
+          else
+            "[#{trace_id}] [ActiveAgent] [#{provider_module}] Stream chunk"
+          end
+        end
+      end
+      # event_log_level :stream_chunk, :debug # Rails 8.1
+
+      # Logs connection failures with service URI and error details.
+      #
+      # @param event [ActiveSupport::Notifications::Event]
+      # @return [void]
+      def connection_error(event)
+        trace_id        = event.payload[:trace_id]
+        provider_module = event.payload[:provider_module]
+        uri_base        = event.payload[:uri_base]
+        exception       = event.payload[:exception]
+        message         = event.payload[:message]
+
+        debug do
+          "[#{trace_id}] [ActiveAgent] [#{provider_module}] Unable to connect to #{uri_base}. Please ensure the service is running. Error: #{exception} - #{message}"
+        end
+      end
+      # event_log_level :connection_error, :debug # Rails 8.1
+
+      private
+
+      # @return [Logger]
+      def logger
+        ActiveAgent::Base.logger
+      end
+    end
+  end
+end
+
+# region log_subscriber_attach
+# Subscribe to both top-level (.active_agent) and provider-level (.provider.active_agent) events
+ActiveAgent::Providers::LogSubscriber.attach_to :active_agent
+ActiveAgent::Providers::LogSubscriber.attach_to :"provider.active_agent"
+# endregion log_subscriber_attach
+
+# Rails 8.1
+# ActiveSupport.event_reporter.subscribe(
+#   ActiveAgent::LogSubscriber.new, &ActiveAgent::LogSubscriber.subscription_filter
+# )

data/lib/active_agent/providers/mock/_types.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative "options"
+require_relative "request"
+require_relative "embedding_request"
+
+module ActiveAgent
+  module Providers
+    module Mock
+      # Type for Request model
+      class RequestType < ActiveModel::Type::Value
+        def cast(value)
+          case value
+          when Request
+            value
+          when Hash
+            Request.new(**value.deep_symbolize_keys)
+          when nil
+            nil
+          else
+            raise ArgumentError, "Cannot cast #{value.class} to Request"
+          end
+        end
+
+        def serialize(value)
+          case value
+          when Request
+            value.serialize
+          when Hash
+            value
+          when nil
+            nil
+          else
+            raise ArgumentError, "Cannot serialize #{value.class}"
+          end
+        end
+
+        def deserialize(value)
+          cast(value)
+        end
+      end
+
+      # Type for EmbeddingRequest model
+      class EmbeddingRequestType < ActiveModel::Type::Value
+        def cast(value)
+          case value
+          when EmbeddingRequest
+            value
+          when Hash
+            EmbeddingRequest.new(**value.deep_symbolize_keys)
+          when nil
+            nil
+          else
+            raise ArgumentError, "Cannot cast #{value.class} to EmbeddingRequest"
+          end
+        end
+
+        def serialize(value)
+          case value
+          when EmbeddingRequest
+            value.serialize
+          when Hash
+            value
+          when nil
+            nil
+          else
+            raise ArgumentError, "Cannot serialize #{value.class}"
+          end
+        end
+
+        def deserialize(value)
+          cast(value)
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/providers/mock/embedding_request.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "active_agent/providers/common/model"
+
+module ActiveAgent
+  module Providers
+    module Mock
+      # Embedding request model for Mock provider.
+      class EmbeddingRequest < Common::BaseModel
+        attribute :model, :string, default: "mock-embedding-model"
+        attribute :input # String or array of strings to embed
+        attribute :encoding_format, :string
+        attribute :dimensions, :integer
+      end
+    end
+  end
+end