oneapm_rpm 1.1.0

data/lib/one_apm/agent/threading/thread_profile.rb

@@ -0,0 +1,155 @@
+# encoding: utf-8
+
+require 'set'
+require 'one_apm/agent/event/worker_loop'
+require 'one_apm/agent/threading/backtrace_node'
+
+# Data structure for representing a thread profile
+
+module OneApm
+  module Agent
+    module Threading
+
+      class ThreadProfile
+
+        attr_reader :profile_id,
+                    :traces,
+                    :sample_period,
+                    :duration,
+                    :poll_count,
+                    :backtrace_count,
+                    :failure_count,
+                    :created_at,
+                    :xray_id,
+                    :command_arguments,
+                    :profile_agent_code
+        attr_accessor :finished_at
+
+        def initialize(command_arguments={})
+          @command_arguments  = command_arguments
+          @profile_id         = command_arguments.fetch('profile_id', -1)
+          @duration           = command_arguments.fetch('duration', 120)
+          @sample_period      = command_arguments.fetch('sample_period', 0.1)
+          @profile_agent_code = command_arguments.fetch('profile_agent_code', false)
+          @xray_id            = command_arguments.fetch('x_ray_id', nil)
+          @finished = false
+
+          @traces = {
+            :agent      => BacktraceRoot.new,
+            :background => BacktraceRoot.new,
+            :other      => BacktraceRoot.new,
+            :request    => BacktraceRoot.new
+          }
+
+          @poll_count = 0
+          @backtrace_count = 0
+          @failure_count = 0
+          @unique_threads = []
+
+          @created_at = Time.now
+        end
+
+        def requested_period
+          @sample_period
+        end
+
+        def increment_poll_count
+          @poll_count += 1
+        end
+
+        def sample_count
+          xray? ? @backtrace_count : @poll_count
+        end
+
+        def xray?
+          !!@xray_id
+        end
+
+        def empty?
+          @backtrace_count == 0
+        end
+
+        def unique_thread_count
+          return 0 if @unique_threads.nil?
+          @unique_threads.length
+        end
+
+        def aggregate(backtrace, bucket, thread)
+          if backtrace.nil?
+            @failure_count += 1
+          else
+            @backtrace_count += 1
+            @traces[bucket].aggregate(backtrace)
+            @unique_threads << thread unless @unique_threads.include?(thread)
+          end
+        end
+
+        def convert_N_trace_nodes_to_arrays(count_to_keep) #THREAD_LOCAL_ACCESS
+          all_nodes = @traces.values.map { |n| n.flattened }.flatten
+
+          OneApm::Agent.instance.stats_engine.
+            tl_record_supportability_metric_count("ThreadProfiler/NodeCount", all_nodes.size)
+
+          all_nodes.sort! do |a, b|
+            # we primarily prefer higher runnable_count
+            comparison = b.runnable_count <=> a.runnable_count
+            # we secondarily prefer lower depth
+            comparison = a.depth          <=> b.depth if comparison == 0
+            # it is thus impossible for any child to preceed their parent
+            comparison
+          end
+
+          all_nodes.each_with_index do |n, i|
+            break if i >= count_to_keep
+            n.mark_for_array_conversion
+          end
+          all_nodes.each_with_index do |n, i|
+            break if i >= count_to_keep
+            n.complete_array_conversion
+          end
+        end
+
+        THREAD_PROFILER_NODES = 20_000
+
+        include OneApm::Coerce
+
+        def generate_traces
+          convert_N_trace_nodes_to_arrays(THREAD_PROFILER_NODES)
+
+          {
+            "OTHER"      => @traces[:other     ].as_array,
+            "REQUEST"    => @traces[:request   ].as_array,
+            "AGENT"      => @traces[:agent     ].as_array,
+            "BACKGROUND" => @traces[:background].as_array
+          }
+        end
+
+        def to_collector_array(encoder)
+          encoded_trace_tree = encoder.encode(generate_traces, :skip_normalization => true)
+          result = [
+            int(self.profile_id),
+            float(self.created_at),
+            float(self.finished_at),
+            int(self.sample_count),
+            encoded_trace_tree,
+            int(self.unique_thread_count),
+            0 # runnable thread count, which we don't track
+          ]
+          result << int(@xray_id) if xray?
+          result
+        end
+
+        def to_log_description
+          id = if xray?
+                 "@xray_id: #{xray_id}"
+               else
+                 "@profile_id: #{profile_id}"
+               end
+
+          "#<ThreadProfile:#{object_id} #{id} @command_arguments=#{@command_arguments.inspect}>"
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/collector/collector/helper.rb

@@ -0,0 +1,139 @@
+module OneApm
+  module Collector
+    class CollectorService
+      module Helper
+
+        def handle_serialization_error(method, e)
+          OneApm::Agent.increment_metric("Supportability/serialization_failure")
+          OneApm::Agent.increment_metric("Supportability/serialization_failure/#{method}")
+          msg = "Failed to serialize #{method} data using #{@marshaller.class.to_s}: #{e.inspect}"
+          error = SerializationError.new(msg)
+          error.set_backtrace(e.backtrace)
+          raise error
+        end
+
+        def record_timing_supportability_metrics(method, start_ts, serialize_finish_ts)
+          serialize_time = serialize_finish_ts && (serialize_finish_ts - start_ts)
+          duration = (Time.now - start_ts).to_f
+          OneApm::Agent.record_metric("Supportability/invoke_remote", duration)
+          OneApm::Agent.record_metric("Supportability/invoke_remote/#{method.to_s}", duration)
+          if serialize_time
+            OneApm::Agent.record_metric("Supportability/invoke_remote_serialize", serialize_time)
+            OneApm::Agent.record_metric("Supportability/invoke_remote_serialize/#{method.to_s}", serialize_time)
+          end
+        end
+
+        # For these metrics, we use the following fields:
+        # call_count           => number of times this remote method was invoked
+        # total_call_time      => total size in bytes of payloads across all invocations
+        # total_exclusive_time => total size in items (e.g. unique metrics, traces, events, etc) across all invocations
+        #
+        # The last field doesn't make sense for all methods (e.g. get_agent_commands),
+        # so we omit it for those methods that don't really take collections
+        # of items as arguments.
+        def record_size_supportability_metrics(method, size_bytes, item_count)
+          metrics = [
+            "Supportability/invoke_remote_size",
+            "Supportability/invoke_remote_size/#{method.to_s}"
+          ]
+          # we may not have an item count, in which case, just record 0 for the exclusive time
+          item_count ||= 0
+          OneApm::Agent.instance.stats_engine.tl_record_unscoped_metrics(metrics, size_bytes, item_count)
+        end
+
+        def log_and_return_response(response)
+          ::OneApm::Agent.logger.debug "Received response, status: #{response.code}, encoding: '#{response['content-encoding']}'"
+
+          case response
+          when Net::HTTPSuccess
+            true # do nothing
+          when Net::HTTPUnauthorized
+            raise LicenseException, 'Invalid license key, please visit support.oneapm.com'
+          when Net::HTTPServiceUnavailable
+            raise ServerConnectionException, "Service unavailable (#{response.code}): #{response.message}"
+          when Net::HTTPGatewayTimeOut
+            raise ServerConnectionException, "Gateway timeout (#{response.code}): #{response.message}"
+          when Net::HTTPRequestEntityTooLarge
+            raise UnrecoverableServerException, '413 Request Entity Too Large'
+          when Net::HTTPUnsupportedMediaType
+            raise UnrecoverableServerException, '415 Unsupported Media Type'
+          else
+            raise ServerConnectionException, "Unexpected response from server (#{response.code}): #{response.message}"
+          end
+
+          response
+        end
+
+        def reset_metric_id_cache
+          @metric_id_cache = {}
+        end
+
+        # ===================== Helpers ==========================
+        # takes an array of arrays of spec and id, adds it into the
+        # metric cache so we can save the collector some work by
+        # sending integers instead of strings the next time around
+        def fill_metric_id_cache(pairs_of_specs_and_ids)
+          Array(pairs_of_specs_and_ids).each do |metric_spec_hash, metric_id|
+            metric_spec = MetricSpec.new(metric_spec_hash['name'],
+                                         metric_spec_hash['scope'])
+            metric_id_cache[metric_spec] = metric_id
+          end
+        rescue => e
+          # If we've gotten this far, we don't want this error to propagate and
+          # make this post appear to have been non-successful, which would trigger
+          # re-aggregation of the same metric data into the next post, so just log
+          OneApm::Agent.logger.error("Failed to fill metric ID cache from response, error details follow ", e)
+        end
+
+        # The collector wants to recieve metric data in a format that's different
+        # from how we store it internally, so this method handles the translation.
+        # It also handles translating metric names to IDs using our metric ID cache.
+        def build_metric_data_array(stats_hash)
+          metric_data_array = []
+          stats_hash.each do |metric_spec, stats|
+            # Omit empty stats as an optimization
+            unless stats.is_reset?
+              metric_id = metric_id_cache[metric_spec]
+              metric_data = if metric_id
+                OneApm::MetricData.new(nil, stats, metric_id)
+              else
+                OneApm::MetricData.new(metric_spec, stats, nil)
+              end
+              metric_data_array << metric_data
+            end
+          end
+          metric_data_array
+        end
+
+        def valid_to_marshal?(data)
+          @marshaller.dump(data)
+          true
+        rescue StandardError, SystemStackError => e
+          OneApm::Agent.logger.warn("Unable to marshal environment report on connect.", e)
+          false
+        end
+
+        # Sets the user agent for connections to the server, to
+        # conform with the HTTP spec and allow for debugging. Includes
+        # the ruby version and also zlib version if available since
+        # that may cause corrupt compression if there is a problem.
+        def user_agent
+          ruby_description = ''
+          # note the trailing space!
+          ruby_description << "(ruby #{::RUBY_VERSION} #{::RUBY_PLATFORM}) " if defined?(::RUBY_VERSION) && defined?(::RUBY_PLATFORM)
+          zlib_version = ''
+          zlib_version << "zlib/#{Zlib.zlib_version}" if defined?(::Zlib) && Zlib.respond_to?(:zlib_version)
+          "OneApm-RubyAgent/#{OneApm::VERSION::STRING} #{ruby_description}#{zlib_version}"
+        end
+
+        private
+
+        # A shorthand for OneApm::Probe.instance
+        def probe
+          OneApm::Probe.instance
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/collector/collector/http_connection.rb

@@ -0,0 +1,254 @@
+module OneApm
+  module Collector
+    class CollectorService
+      module HttpConnection
+
+        # These include Errno connection errors, and all indicate that the
+        # underlying TCP connection may be in a bad state.
+        CONNECTION_ERRORS = [Timeout::Error, EOFError, SystemCallError, SocketError].freeze
+
+        def invoke_remote(method, payload = [], options = {})
+          start_ts = Time.now
+
+          data = nil
+          begin
+            data = @marshaller.dump(payload, options)
+            if method != :connect
+              data = data.gsub(/(Controller|HttpDispatcher)/, "WebTransaction")
+            end
+          rescue StandardError, SystemStackError => e
+            handle_serialization_error(method, e)
+          end
+          serialize_finish_ts = Time.now
+
+          data, encoding = compress_request_if_needed(data)
+          size = data.size
+
+          uri = remote_method_uri(method, @marshaller.format)
+          full_uri = "#{@collector}#{uri}"
+
+          @audit_logger.log_request(full_uri, payload, @marshaller)
+          response = send_request(:data      => data,
+                                  :uri       => uri,
+                                  :encoding  => encoding,
+                                  :collector => @collector)
+          @marshaller.load(decompress_response(response))
+        ensure
+          record_timing_supportability_metrics(method, start_ts, serialize_finish_ts)
+          if size
+            record_size_supportability_metrics(method, size, options[:item_count])
+          end
+        end
+
+        # The path on the server that we should post our data to
+        def remote_method_uri(method, format)
+          params = {'run_id' => @agent_id, 'marshal_format' => format}
+          uri = "/tpm/agent.do?PROTOCOL_VERSION=#{PROTOCOL_VERSION}&license_key=#{@license_key}&method=#{method}"
+          uri << '&' + params.map do |k,v|
+            next unless v
+            "#{k}=#{v}"
+          end.compact.join('&')
+          uri
+        end
+
+        # Posts to the specified server
+        #
+        # Options:
+        #  - :uri => the path to request on the server (a misnomer of
+        #              course)
+        #  - :encoding => the encoding to pass to the server
+        #  - :collector => a URI object that responds to the 'name' method
+        #                    and returns the name of the collector to
+        #                    contact
+        #  - :data => the data to send as the body of the request
+        def send_request(opts)
+          request = Net::HTTP::Post.new(opts[:uri], 'CONTENT-ENCODING' => opts[:encoding], 'HOST' => opts[:collector].name)
+          request['user-agent'] = user_agent
+          request.content_type = "application/octet-stream"
+          request.body = opts[:data]
+
+          response     = nil
+          attempts     = 0
+          max_attempts = 2
+
+          begin
+            attempts += 1
+            conn = http_connection
+            ::OneApm::Agent.logger.debug "Sending request to #{opts[:collector]}#{opts[:uri]}"
+            OneApm::TimerLib.timeout(@request_timeout) do
+              response = conn.request(request)
+            end
+          rescue *CONNECTION_ERRORS => e
+            close_shared_connection
+            if attempts < max_attempts
+              ::OneApm::Agent.logger.debug("Retrying request to #{opts[:collector]}#{opts[:uri]} after #{e}")
+              retry
+            else
+              raise ServerConnectionException, "Recoverable error talking to #{@collector} after #{attempts} attempts: #{e}"
+            end
+          end
+
+          log_and_return_response response
+        end
+
+        def session(&block)
+          raise ArgumentError, "#{self.class}#shared_connection must be passed a block" unless block_given?
+
+          begin
+            t0 = Time.now
+            @in_session = true
+            if OneApm::Agent.config[:aggressive_keepalive]
+              session_with_keepalive(&block)
+            else
+              session_without_keepalive(&block)
+            end
+          rescue *CONNECTION_ERRORS => e
+            elapsed = Time.now - t0
+            raise OneApm::ServerConnectionException, "Recoverable error connecting to #{@collector} after #{elapsed} seconds: #{e}"
+          ensure
+            @in_session = false
+          end
+        end
+
+        def session_with_keepalive(&block)
+          establish_shared_connection
+          block.call
+        end
+
+        def session_without_keepalive(&block)
+          begin
+            establish_shared_connection
+            block.call
+          ensure
+            close_shared_connection
+          end
+        end
+
+        def http_connection
+          if @in_session
+            establish_shared_connection
+          else
+            create_http_connection
+          end
+        end
+
+        def establish_shared_connection
+          unless @shared_tcp_connection
+            @shared_tcp_connection = create_and_start_http_connection
+          end
+          @shared_tcp_connection
+        end
+
+        def close_shared_connection
+          if @shared_tcp_connection
+            ::OneApm::Agent.logger.debug("Closing shared TCP connection to #{@shared_tcp_connection.address}:#{@shared_tcp_connection.port}")
+            @shared_tcp_connection.finish if @shared_tcp_connection.started?
+            @shared_tcp_connection = nil
+          end
+        end
+
+        def setup_connection_timeouts(conn)
+          # We use Timeout explicitly instead of this
+          conn.read_timeout = nil
+
+          if conn.respond_to?(:keep_alive_timeout) && OneApm::Agent.config[:aggressive_keepalive]
+            conn.keep_alive_timeout = OneApm::Agent.config[:keep_alive_timeout]
+          end
+        end
+
+        def create_and_start_http_connection
+          conn = create_http_connection
+          start_connection(conn)
+          conn
+        end
+
+        def start_connection(conn)
+          OneApm::Agent.logger.debug("Opening TCP connection to #{conn.address}:#{conn.port}")
+          OneApm::TimerLib.timeout(@request_timeout) { conn.start }
+          conn
+        end
+
+        # Return the Net::HTTP with proxy configuration given the OneApm::Support::Server object.
+        def create_http_connection
+          # Proxy returns regular HTTP if @proxy_host is nil (the default)
+          http_class = Net::HTTP::Proxy(proxy_server.name, proxy_server.port,
+                                        proxy_server.user, proxy_server.password)
+
+          conn = http_class.new((@collector.ip || @collector.name), @collector.port)
+          setup_connection_for_ssl(conn) if Agent.config[:ssl]
+          setup_connection_timeouts(conn)
+
+          ::OneApm::Agent.logger.debug("Created net/http handle to #{conn.address}:#{conn.port}")
+          conn
+        end
+
+        def setup_connection_for_ssl(conn)
+          # Jruby 1.6.8 requires a gem for full ssl support and will throw
+          # an error when use_ssl=(true) is called and jruby-openssl isn't
+          # installed
+          conn.use_ssl     = true
+          conn.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          conn.cert_store  = ssl_cert_store
+        rescue StandardError, LoadError
+          msg = "Agent is configured to use SSL, but SSL is not available in the environment. "
+          msg << "Either disable SSL in the agent configuration, or install SSL support."
+          raise UnrecoverableAgentException.new(msg)
+        end
+
+        def ssl_cert_store
+          path = cert_file_path
+          if !@ssl_cert_store || path != @cached_cert_store_path
+            ::OneApm::Agent.logger.debug("Creating SSL certificate store from file at #{path}")
+            @ssl_cert_store = OpenSSL::X509::Store.new
+            @ssl_cert_store.add_file(path)
+            @cached_cert_store_path = path
+          end
+          @ssl_cert_store
+        end
+
+        # The path to the certificate file used to verify the SSL
+        # connection if verify_peer is enabled
+        def cert_file_path
+          if path_override = OneApm::Agent.config[:ca_bundle_path]
+            OneApm::Agent.logger.warn("Couldn't find CA bundle from configured ca_bundle_path: #{path_override}") unless File.exists? path_override
+            path_override
+          else
+            File.expand_path(File.join(probe.oneapm_root, 'config', 'cert', 'cacert.pem'))
+          end
+        end
+
+        # We do not compress if content is smaller than 64kb.  There are
+        # problems with bugs in Ruby in some versions that expose us
+        # to a risk of segfaults if we compress aggressively.
+        def compress_request_if_needed(data)
+          encoding = 'identity'
+          if data.size > 64 * 1024
+            data = OneApm::Support::Encoders::Compressed.encode(data)
+            encoding = 'deflate'
+          end
+          check_post_size(data)
+          [data, encoding]
+        end
+
+        # Decompresses the response from the server, if it is gzip
+        # encoded, otherwise returns it verbatim
+        def decompress_response(response)
+          if response['content-encoding'] == 'gzip'
+            Zlib::GzipReader.new(StringIO.new(response.body)).read
+          else
+            response.body
+          end
+        end
+
+        # Raises an UnrecoverableServerException if the post_string is longer
+        # than the limit configured in the probe object
+        def check_post_size(post_string)
+          return if post_string.size < Agent.config[:post_size_limit]
+          ::OneApm::Agent.logger.debug "Tried to send too much data: #{post_string.size} bytes"
+          raise UnrecoverableServerException.new('413 Request Entity Too Large')
+        end
+
+      end
+    end
+  end
+end