oneapm_rpm 1.1.0

data/lib/one_apm/collector/collector/server_methods.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require 'resolv'
+require 'one_apm/support/server'
+
+module OneApm
+  module Collector
+    class CollectorService
+      module ServerMethods
+
+        def server
+          @remote_server ||= server_from_host
+        end
+
+        # a new instances of the proxy server - this passes through if
+        # there is no proxy, otherwise it has proxy configuration
+        # information pulled from the config file
+        def proxy_server
+          @proxy_server ||= OneApm::Support::ProxyServer.new(Agent.config[:proxy_host],
+                                                             Agent.config[:proxy_port],
+                                                             Agent.config[:proxy_user],
+                                                             Agent.config[:proxy_pass])
+        end
+
+        # turns a hostname into an ip address and returns a
+        # OneApm::Support::Server that contains the configuration info
+        def server_from_host(hostname=nil)
+          host = hostname || Agent.config[:host]
+
+          # if the host is not an IP address, turn it into one
+          OneApm::Support::Server.new(host, Agent.config[:port], convert_to_ip_address(host))
+        end
+
+        # Check to see if we need to look up the IP address
+        # If it's an IP address already, we pass it through.
+        # If it's nil, or localhost, we don't bother.
+        # Otherwise, use `resolve_ip_address` to find one
+        def convert_to_ip_address(host)
+          # here we leave it as a host name since the cert verification
+          # needs it in host form
+          return host if Agent.config[:ssl]
+          # We won't talk directly to the host, so no need to resolve if proxy configured
+          return host if Agent.config[:proxy_host]
+          return nil if host.nil? || host.downcase == "localhost"
+          ip = resolve_ip_address(host)
+
+          OneApm::Agent.logger.debug "Resolved #{host} to #{ip}"
+          ip
+        end
+
+        # Look up the ip address of the host using the pure ruby lookup
+        # to prevent blocking.  If that fails, fall back to the regular
+        # IPSocket library.  Return nil if we can't find the host ip
+        # address and don't have a good default.
+        def resolve_ip_address(host)
+          Resolv.getaddress(host)
+        rescue => e
+          OneApm::Agent.logger.warn("DNS Error caching IP address:", e)
+          begin
+            OneApm::Agent.logger.debug("Trying native DNS lookup since Resolv failed")
+            IPSocket.getaddress(host)
+          rescue => e
+            OneApm::Agent.logger.error("Could not look up server address: #{e}")
+            nil
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/collector/collector_service.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+
+require 'zlib'
+require 'one_apm/logger/audit_logger'
+require 'one_apm/support/encoders'
+require 'one_apm/support/marshaller'
+require 'one_apm/support/json_marshaller'
+require 'one_apm/collector/collector/server_methods'
+require 'one_apm/collector/collector/http_connection'
+require 'one_apm/collector/collector/helper'
+
+module OneApm
+  module Collector
+    class CollectorService
+      include OneApm::Collector::CollectorService::Helper
+      include OneApm::Collector::CollectorService::ServerMethods
+      include OneApm::Collector::CollectorService::HttpConnection
+
+      PROTOCOL_VERSION = 1
+
+      attr_accessor :request_timeout, :agent_id
+      attr_reader :collector,:marshaller, :metric_id_cache
+
+      def initialize(license_key = nil, collector = nil)
+        @license_key = license_key || Agent.config[:license_key]
+        @collector = collector || server
+        @request_timeout = Agent.config[:timeout]
+        @metric_id_cache = {}
+        @audit_logger = ::OneApm::Agent::AuditLogger.new
+
+        Agent.config.register_callback(:'audit_log.enabled') do |enabled|
+          @audit_logger.enabled = enabled
+        end
+
+        Agent.config.register_callback(:ssl) do |ssl|
+          if !ssl
+            OneApm::Agent.logger.warn("Agent is configured not to use SSL when communicating with OneApm's servers")
+          else
+            OneApm::Agent.logger.debug("Agent is configured to use SSL")
+          end
+        end
+
+        Agent.config.register_callback(:marshaller) do |marshaller|
+          @marshaller = OneApm::Support::JsonMarshaller.new
+        end
+      end
+
+      def get_redirect_host
+        invoke_remote(:get_redirect_host)
+      end
+
+      def connect(settings={})
+        if host = get_redirect_host
+          @collector = server_from_host(host)
+        end
+        response = invoke_remote(:connect, [settings])
+        @agent_id = response['agent_run_id']
+        response
+      end
+
+      def shutdown(time)
+        invoke_remote(:shutdown, [@agent_id, time.to_i]) if @agent_id
+      end
+
+      def metric_data(stats_hash)
+        timeslice_start = stats_hash.started_at
+        timeslice_end   = stats_hash.harvested_at || Time.now
+
+        metric_data_array = build_metric_data_array(stats_hash)
+
+        result = invoke_remote(
+          :metric_data,
+          [@agent_id, timeslice_start.to_f, timeslice_end.to_f, metric_data_array],
+          :item_count => metric_data_array.size
+        )
+
+        fill_metric_id_cache(result)
+        result
+      end
+
+      def error_data(unsent_errors)
+        invoke_remote(:error_data, [@agent_id, unsent_errors], :item_count => unsent_errors.size)
+      end
+
+      def transaction_sample_data(traces)
+        invoke_remote(:transaction_sample_data, [@agent_id, traces], :item_count => traces.size)
+      end
+
+      def sql_trace_data(sql_traces)
+        invoke_remote(:sql_trace_data, [sql_traces], :item_count => sql_traces.size)
+      end
+
+      def profile_data(profile)
+        invoke_remote(:profile_data, [@agent_id, profile], :skip_normalization => true) || ''
+      end
+
+      def get_agent_commands
+        invoke_remote(:get_agent_commands, [@agent_id])
+      end
+
+      def agent_command_results(results)
+        invoke_remote(:agent_command_results, [@agent_id, results])
+      end
+
+      def get_xray_metadata(xray_ids)
+        invoke_remote(:get_xray_metadata, [@agent_id, *xray_ids])
+      end
+
+      def analytic_event_data(data)
+        invoke_remote(:analytic_event_data, [@agent_id, data], :item_count => data.size)
+      end
+
+      def custom_event_data(data)
+        invoke_remote(:custom_event_data, [@agent_id, data], :item_count => data.size)
+      end
+
+      def utilization_data(data)
+        invoke_remote(:utilization_data, data)
+      end
+
+    end
+  end
+end

data/lib/one_apm/collector/commands/agent_command.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+module OneApm
+  module Collector
+    module Commands
+      class AgentCommand
+        attr_reader :id, :name, :arguments
+
+        def initialize(raw_collector_command)
+          @id        = raw_collector_command[0]
+          @name      = raw_collector_command[1]["name"]
+          @arguments = raw_collector_command[1]["arguments"]
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/collector/commands/thread_profiler_session.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+require 'one_apm/agent/threading/agent_thread'
+require 'one_apm/agent/threading/backtrace_service'
+require 'one_apm/agent/threading/thread_profile'
+
+module OneApm
+  module Collector
+    module Commands
+
+      class ThreadProfilerSession
+
+        def initialize(backtrace_service)
+          @backtrace_service = backtrace_service
+        end
+
+        def handle_start_command(agent_command)
+          raise_unsupported_error unless OneApm::Agent::Threading::BacktraceService.is_supported?
+          raise_thread_profiler_disabled unless enabled?
+          raise_already_started_error if running?
+          start(agent_command)
+        end
+
+        def handle_stop_command(agent_command)
+          report_data = agent_command.arguments.fetch("report_data", true)
+          stop(report_data)
+        end
+
+        def start(agent_command)
+          OneApm::Agent.logger.debug("Starting Thread Profiler.")
+          profile = @backtrace_service.subscribe(
+            OneApm::Agent::Threading::BacktraceService::ALL_TRANSACTIONS,
+            agent_command.arguments
+          )
+
+          @started_at = Time.now
+          @duration = profile.duration if profile
+        end
+
+        def stop(report_data)
+          return unless running?
+          OneApm::Agent.logger.debug("Stopping Thread Profiler.")
+          @finished_profile = @backtrace_service.harvest(OneApm::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+          @backtrace_service.unsubscribe(OneApm::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+          @finished_profile = nil if !report_data
+        end
+
+        def harvest
+          OneApm::Agent.logger.debug("Harvesting from Thread Profiler #{@finished_profile.to_log_description unless @finished_profile.nil?}")
+          profile = @finished_profile
+          @backtrace_service.profile_agent_code = false
+          @finished_profile = nil
+          @started_at = nil
+          profile
+        end
+
+        def enabled?
+          OneApm::Agent.config[:'thread_profiler.enabled']
+        end
+
+        def running?
+          @backtrace_service.subscribed?(OneApm::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+        end
+
+        def ready_to_harvest?
+          past_time? || stopped?
+        end
+
+        def past_time?
+          @started_at && (Time.now > @started_at + @duration)
+        end
+
+        def stopped?
+          !!@finished_profile
+        end
+
+        private
+
+        def raise_command_error(msg)
+          raise OneApm::Collector::AgentCommandRouter::AgentCommandError.new(msg)
+        end
+
+        def raise_unsupported_error
+          msg = <<-EOF
+            Thread profiling is only supported on 1.9.2 and greater versions of Ruby.
+            We detected running agents capable of profiling, but the profile started with
+            an agent running Ruby #{RUBY_VERSION}.
+
+            Profiling again might select an appropriate agent, but we recommend running a
+            consistent version of Ruby across your application for better results.
+          EOF
+          raise_command_error(msg)
+        end
+
+        def raise_thread_profiler_disabled
+          msg = "Not starting Thread Profiler because of config 'thread_profiler.enabled' = #{enabled?}"
+          raise_command_error(msg)
+        end
+
+        def raise_already_started_error
+          msg = "Profile already in progress. Ignoring agent command to start another."
+          raise_command_error(msg)
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/collector/commands/xray_session.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'forwardable'
+
+module OneApm
+  module Collector
+    module Commands
+      class XraySession
+        extend Forwardable
+
+        attr_reader :id, :command_arguments
+        attr_reader :xray_session_name, :key_transaction_name,
+                    :requested_trace_count, :duration, :sample_period
+
+        def initialize(command_arguments)
+          @command_arguments     = command_arguments
+          @id                    = command_arguments.fetch("x_ray_id", nil)
+          @xray_session_name     = command_arguments.fetch("xray_session_name", "")
+          @key_transaction_name  = command_arguments.fetch("key_transaction_name", "")
+          @requested_trace_count = command_arguments.fetch("requested_trace_count", 100)
+          @duration              = command_arguments.fetch("duration", 86400)
+          @sample_period         = command_arguments.fetch("sample_period", 0.1)
+          @run_profiler          = command_arguments.fetch("run_profiler", true)
+        end
+
+        def active?
+          @active
+        end
+
+        def run_profiler?
+          @run_profiler && OneApm::Agent.config[:'xray_session.allow_profiles']
+        end
+
+        def activate
+          @active = true
+          @start_time = Time.now
+        end
+
+        def deactivate
+          @active = false
+        end
+
+        def requested_period
+          @sample_period
+        end
+
+        def finished?
+          @start_time + @duration < Time.now
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/collector/commands/xray_session_collection.rb

@@ -0,0 +1,156 @@
+# encoding: utf-8
+
+require 'forwardable'
+require 'thread'
+require 'one_apm/collector/commands/xray_session'
+
+module OneApm
+  module Collector
+    module Commands
+      class XraySessionCollection
+        extend Forwardable
+
+        def initialize(backtrace_service, event_listener)
+          @backtrace_service = backtrace_service
+
+          # This lock protects access to the sessions hash, but it's expected
+          # that individual session objects within the hash will be manipulated
+          # outside the lock.  This is safe because manipulation of the session
+          # objects is expected from only a single thread (the harvest thread)
+          @sessions_lock = Mutex.new
+          @sessions = {}
+
+          if event_listener
+            event_listener.subscribe(:before_harvest, &method(:cleanup_finished_sessions))
+          end
+        end
+
+        def handle_active_xray_sessions(agent_command)
+          # If X-Rays are disabled, just be quiet about it and don't start the
+          # command. Other hosts might be running the X-Ray, so we don't need
+          # to bark on every get_agent_commands.
+          if !OneApm::Agent.config[:'xray_session.enabled']
+            OneApm::Agent.logger.debug("Not responding to X-Ray command because of config 'xray_session.enabled' = #{OneApm::Agent.config[:'xray_session.enabled']}")
+            return
+          end
+
+          incoming_ids = agent_command.arguments["xray_ids"]
+          deactivate_for_incoming_sessions(incoming_ids)
+          activate_sessions(incoming_ids)
+        end
+
+        def session_id_for_transaction_name(name)
+          @sessions_lock.synchronize do
+            @sessions.keys.find { |id| @sessions[id].key_transaction_name == name }
+          end
+        end
+
+        NO_PROFILES = [].freeze
+
+        def harvest_thread_profiles
+          return NO_PROFILES unless OneApm::Agent::Threading::BacktraceService.is_supported?
+
+          profiles = active_thread_profiling_sessions.map do |session|
+            OneApm::Agent.logger.debug("Harvesting profile for X-Ray session #{session.inspect}")
+            @backtrace_service.harvest(session.key_transaction_name)
+          end
+          profiles.reject! {|p| p.empty?}
+          profiles.compact
+        end
+
+        def stop_all_sessions
+          deactivate_for_incoming_sessions([])
+        end
+
+        def cleanup_finished_sessions
+          finished_session_ids.each do |id|
+            OneApm::Agent.logger.debug("Finished X-Ray session #{id} by duration. Removing it from active sessions.")
+            remove_session_by_id(id)
+          end
+        end
+
+
+        ### Internals
+
+        def one_apm_service
+          OneApm::Agent.instance.service
+        end
+
+        # These are unsynchonized and should only be used for testing
+        def_delegators :@sessions, :[], :include?
+
+        def active_thread_profiling_sessions
+          @sessions_lock.synchronize do
+            @sessions.values.select { |s| s.active? && s.run_profiler? }
+          end
+        end
+
+        ### Session activation
+
+        def activate_sessions(incoming_ids)
+          lookup_metadata_for(ids_to_activate(incoming_ids)).each do |raw|
+            add_session(XraySession.new(raw))
+          end
+        end
+
+        def ids_to_activate(incoming_ids)
+          @sessions_lock.synchronize { incoming_ids - @sessions.keys }
+        end
+
+        # Please don't hold the @sessions_lock across me! Calling the service
+        # is time-consuming, and will block request threads. Which is rude.
+        def lookup_metadata_for(ids_to_activate)
+          return [] if ids_to_activate.empty?
+
+          OneApm::Agent.logger.debug("Retrieving metadata for X-Ray sessions #{ids_to_activate.inspect}")
+          one_apm_service.get_xray_metadata(ids_to_activate)
+        end
+
+        def add_session(session)
+          OneApm::Agent.logger.debug("Adding X-Ray session #{session.inspect}")
+          OneApm::Agent.increment_metric("Supportability/XraySessions/Starts")
+
+          @sessions_lock.synchronize { @sessions[session.id] = session }
+
+          session.activate
+          if session.run_profiler?
+            @backtrace_service.subscribe(session.key_transaction_name, session.command_arguments)
+          end
+        end
+
+        ### Session deactivation
+
+        def deactivate_for_incoming_sessions(incoming_ids)
+          ids_to_remove(incoming_ids).each do |session_id|
+            remove_session_by_id(session_id)
+          end
+        end
+
+        def ids_to_remove(incoming_ids)
+          @sessions_lock.synchronize { @sessions.keys - incoming_ids }
+        end
+
+        def remove_session_by_id(id)
+          session = @sessions_lock.synchronize { @sessions.delete(id) }
+
+          if session
+            OneApm::Agent.logger.debug("Removing X-Ray session #{session.inspect}")
+            OneApm::Agent.increment_metric("Supportability/XraySessions/Stops")
+
+            if session.run_profiler?
+              @backtrace_service.unsubscribe(session.key_transaction_name)
+            end
+            session.deactivate
+          end
+        end
+
+        def finished_session_ids
+          @sessions_lock.synchronize do
+            @sessions.map{|k, s| k if s.finished?}.compact
+          end
+        end
+
+      end
+    end
+  end
+end