ghazel-newrelic_rpm 3.1.0.1 → 3.4.0.2

data/lib/new_relic/agent/new_relic_service.rb

@@ -0,0 +1,221 @@
+module NewRelic
+  module Agent
+    class NewRelicService
+      # Specifies the version of the agent's communication protocol with
+      # the NewRelic hosted site.
+      
+      PROTOCOL_VERSION = 8
+      # 14105: v8 (tag 2.10.3)
+      # (no v7)
+      # 10379: v6 (not tagged)
+      # 4078:  v5 (tag 2.5.4)
+      # 2292:  v4 (tag 2.3.6)
+      # 1754:  v3 (tag 2.3.0)
+      # 534:   v2 (shows up in 2.1.0, our first tag)
+      
+      attr_accessor :request_timeout
+      attr_reader :collector
+      attr_accessor :agent_id
+      
+      def initialize(license_key=control.license_key, collector=control.server)
+        @license_key = license_key
+        @collector = collector
+        @request_timeout = NewRelic::Control.instance.fetch('timeout', 2 * 60)
+      end
+      
+      def connect(settings={})
+        host = get_redirect_host
+        @collector.name = host if host
+        response = invoke_remote(:connect, settings)
+        @agent_id = response['agent_run_id']
+        response
+      end
+
+      def get_redirect_host
+        invoke_remote(:get_redirect_host)
+      end
+
+      def shutdown(time)
+        invoke_remote(:shutdown, @agent_id, time) if @agent_id
+      end
+
+      def metric_data(last_harvest_time, now, unsent_timeslice_data)
+        invoke_remote(:metric_data, @agent_id, last_harvest_time, now,
+                      unsent_timeslice_data)
+      end
+      
+      def error_data(unsent_errors)
+        invoke_remote(:error_data, @agent_id, unsent_errors)
+      end
+
+      def transaction_sample_data(traces)
+        invoke_remote(:transaction_sample_data, @agent_id, traces)
+      end
+
+      def sql_trace_data(sql_traces)
+        invoke_remote(:sql_trace_data, sql_traces)
+      end
+
+      private
+      
+      # A shorthand for NewRelic::Control.instance
+      def control
+        NewRelic::Control.instance
+      end
+      
+      # Shorthand to the NewRelic::Agent.logger method
+      def log
+        NewRelic::Agent.logger
+      end
+      
+      # The path on the server that we should post our data to
+      def remote_method_uri(method)
+        uri = "/agent_listener/#{PROTOCOL_VERSION}/#{@license_key}/#{method}"
+        uri << "?run_id=#{@agent_id}" if @agent_id
+        uri
+      end
+      
+      # send a message via post to the actual server. This attempts
+      # to automatically compress the data via zlib if it is large
+      # enough to be worth compressing, and handles any errors the
+      # server may return
+      def invoke_remote(method, *args)
+        now = Time.now
+        #determines whether to zip the data or send plain
+        post_data, encoding = compress_data(args)
+        
+        response = send_request(:uri       => remote_method_uri(method),
+                                :encoding  => encoding,
+                                :collector => @collector,
+                                :data      => post_data)
+
+        # raises the right exception if the remote server tells it to die
+        return check_for_exception(response)
+      rescue NewRelic::Agent::ForceRestartException => e
+        log.info e.message
+        raise
+      rescue SystemCallError, SocketError => e
+        # These include Errno connection errors
+        raise NewRelic::Agent::ServerConnectionException, "Recoverable error connecting to the server: #{e}"
+      ensure
+        NewRelic::Agent.instance.stats_engine.get_stats_no_scope('Supportability/invoke_remote').record_data_point((Time.now - now).to_f)
+        NewRelic::Agent.instance.stats_engine.get_stats_no_scope('Supportability/invoke_remote/' + method.to_s).record_data_point((Time.now - now).to_f)
+      end
+
+      # This method handles the compression of the request body that
+      # we are going to send to the server
+      #
+      # We currently optimize for CPU here since we get roughly a 10x
+      # reduction in message size with this, and CPU overhead is at a
+      # premium. For extra-large posts, we use the higher compression
+      # since otherwise it actually errors out.
+      #
+      # We do not compress if content is smaller than 64kb.  There are
+      # problems with bugs in Ruby in some versions that expose us
+      # to a risk of segfaults if we compress aggressively.
+      #
+      # medium payloads get fast compression, to save CPU
+      # big payloads get all the compression possible, to stay under
+      # the 2,000,000 byte post threshold
+      def compress_data(object)
+        dump = Marshal.dump(object)
+
+        dump_size = dump.size
+
+        return [dump, 'identity'] if dump_size < (64*1024)
+
+        compressed_dump = Zlib::Deflate.deflate(dump, Zlib::DEFAULT_COMPRESSION)
+
+        # this checks to make sure mongrel won't choke on big uploads
+        check_post_size(compressed_dump)
+
+        [compressed_dump, 'deflate']
+      end
+
+      # Raises a PostTooBigException if the post_string is longer
+      # than the limit configured in the control object
+      def check_post_size(post_string)
+        # TODO: define this as a config option on the server side
+        return if post_string.size < control.post_size_limit
+        log.warn "Tried to send too much data: #{post_string.size} bytes"
+        raise PostTooBigException
+      end
+
+      # Posts to the specified server
+      #
+      # Options:
+      #  - :uri => the path to request on the server (a misnomer of
+      #              course)
+      #  - :encoding => the encoding to pass to the server
+      #  - :collector => a URI object that responds to the 'name' method
+      #                    and returns the name of the collector to
+      #                    contact
+      #  - :data => the data to send as the body of the request
+      def send_request(opts)
+        request = Net::HTTP::Post.new(opts[:uri], 'CONTENT-ENCODING' => opts[:encoding], 'HOST' => opts[:collector].name)
+        request['user-agent'] = user_agent
+        request.content_type = "application/octet-stream"
+        request.body = opts[:data]
+        
+        log.debug "Connect to #{opts[:collector]}#{opts[:uri]}"
+        
+        response = nil
+        http = control.http_connection(@collector)
+        http.read_timeout = nil
+        begin
+          NewRelic::TimerLib.timeout(@request_timeout) do
+            response = http.request(request)
+          end
+        rescue Timeout::Error
+          log.warn "Timed out trying to post data to New Relic (timeout = #{@request_timeout} seconds)" unless @service.request_timeout < 30
+          raise
+        end
+        if response.is_a? Net::HTTPServiceUnavailable
+          raise NewRelic::Agent::ServerConnectionException, "Service unavailable (#{response.code}): #{response.message}"
+        elsif response.is_a? Net::HTTPGatewayTimeOut
+          log.debug("Timed out getting response: #{response.message}")
+          raise Timeout::Error, response.message
+        elsif response.is_a? Net::HTTPRequestEntityTooLarge
+          raise PostTooBigException
+        elsif !(response.is_a? Net::HTTPSuccess)
+          raise NewRelic::Agent::ServerConnectionException, "Unexpected response from server (#{response.code}): #{response.message}"
+        end
+        response
+      end
+
+      # Decompresses the response from the server, if it is gzip
+      # encoded, otherwise returns it verbatim
+      def decompress_response(response)
+        if response['content-encoding'] != 'gzip'
+          log.debug "Uncompressed content returned"
+          return response.body
+        end
+        log.debug "Decompressing return value"
+        i = Zlib::GzipReader.new(StringIO.new(response.body))
+        i.read
+      end
+
+      # unmarshals the response and raises it if it is an exception,
+      # so we can handle it in nonlocally
+      def check_for_exception(response)
+        dump = decompress_response(response)
+        value = Marshal.load(dump)
+        raise value if value.is_a? Exception
+        value
+      end
+
+      # Sets the user agent for connections to the server, to
+      # conform with the HTTP spec and allow for debugging. Includes
+      # the ruby version and also zlib version if available since
+      # that may cause corrupt compression if there is a problem.
+      def user_agent
+        ruby_description = ''
+        # note the trailing space!
+        ruby_description << "(ruby #{::RUBY_VERSION} #{::RUBY_PLATFORM}) " if defined?(::RUBY_VERSION) && defined?(::RUBY_PLATFORM)
+        zlib_version = ''
+        zlib_version << "zlib/#{Zlib.zlib_version}" if defined?(::Zlib) && Zlib.respond_to?(:zlib_version)
+        "NewRelic-RubyAgent/#{NewRelic::VERSION::STRING} #{ruby_description}#{zlib_version}"
+      end
+    end
+  end
+end

data/lib/new_relic/agent/pipe_channel_manager.rb

@@ -0,0 +1,161 @@
+module NewRelic
+  module Agent
+    module PipeChannelManager
+      extend self
+      
+      def register_report_channel(id)
+        listener.register_pipe(id)
+      end
+
+      def channels
+        listener.pipes
+      end
+      
+      def listener
+        @listener ||= Listener.new
+      end
+      
+      class Pipe
+        attr_accessor :in, :out
+        attr_reader :last_read
+        
+        def initialize
+          @out, @in = IO.pipe
+          if defined?(::Encoding::ASCII_8BIT)
+            @in.set_encoding(::Encoding::ASCII_8BIT)
+          end
+          @last_read = Time.now
+        end
+        
+        def close
+          @out.close unless @out.closed?
+          @in.close unless @in.closed?
+        end
+
+        def write(data)
+          @out.close unless @out.closed?
+          @in << Marshal.dump(data) + "\n\n"
+        end
+        
+        def read
+          @in.close unless @in.closed?
+          @last_read = Time.now
+          @out.gets("\n\n")
+        end
+
+        def closed?
+          @out.closed? && @in.closed?
+        end
+      end
+      
+      class Listener
+        attr_reader   :thread
+        attr_accessor :pipes, :timeout, :select_timeout
+
+        def initialize
+          @pipes = {}
+          @timeout = 360
+          @select_timeout = 60
+        end
+
+        def register_pipe(id)
+          @pipes[id] = Pipe.new
+          wake.in << '.'
+        end
+
+        def start
+          return if @started == true
+          @started = true
+          @thread = Thread.new do
+            loop do
+              clean_up_pipes
+              pipes_to_listen_to = @pipes.values.map{|pipe| pipe.out} + [wake.out]
+              if ready = IO.select(pipes_to_listen_to, [], [], @select_timeout) 
+                pipe = ready[0][0]
+                if pipe == wake.out
+                  pipe.read(1)
+                else
+                  merge_data_from_pipe(pipe)
+                end
+              end
+              
+              break if !should_keep_listening?
+            end
+          end
+          @thread #.abort_on_exception = true
+          sleep 0.001 # give time for the thread to spawn
+        end
+        
+        def stop
+          return unless @started == true
+          @started = false
+          wake.in << '.' unless wake.in.closed?
+          @thread.join # make sure we wait for the thread to exit
+          close_all_pipes
+          @wake.close
+          @wake = nil
+        end
+
+        def close_all_pipes
+          @pipes.each do |id, pipe|
+            pipe.close if pipe
+          end
+          @pipes = {}
+        end
+        
+        def wake
+          @wake ||= Pipe.new
+        end
+
+        def started?
+          @started
+        end
+
+        protected
+
+        def merge_data_from_pipe(pipe_handle)
+          pipe = find_pipe_for_handle(pipe_handle)
+          got = pipe.read
+          
+          if got && !got.empty?
+            payload = unmarshal(got)
+            if payload == 'EOF'
+              pipe.close
+            else
+              NewRelic::Agent.agent.merge_data_from([payload[:stats],
+                                                     payload[:transaction_traces],
+                                                     payload[:error_traces]])
+            end
+          end
+        end
+
+        def unmarshal(data)
+          if NewRelic::LanguageSupport.broken_gc?
+            NewRelic::LanguageSupport.with_disabled_gc do
+              Marshal.load(data)
+            end
+          else
+            Marshal.load(data)
+          end
+        end
+        
+        def should_keep_listening?
+          @started || @pipes.values.find{|pipe| !pipe.in.closed?}
+        end
+        
+        def clean_up_pipes
+          @pipes.values.each do |pipe|
+            if pipe.last_read.to_f + @timeout < Time.now.to_f
+              pipe.close unless pipe.closed?
+            end
+          end
+          @pipes.reject! {|id, pipe| pipe.out.closed? }
+        end
+
+        def find_pipe_for_handle(out_handle)
+          @pipes.values.find{|pipe| pipe.out == out_handle }
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/pipe_service.rb

@@ -0,0 +1,54 @@
+module NewRelic
+  module Agent
+    class PipeService
+      attr_reader :channel_id, :buffer
+      attr_accessor :request_timeout, :agent_id, :collector
+      
+      def initialize(channel_id)
+        @channel_id = channel_id
+        @collector = NewRelic::Control::Server.new(:name => 'parent',
+                                                   :port => 0)
+      end
+      
+      def connect(config)
+        nil
+      end
+      
+      def metric_data(last_harvest_time, now, unsent_timeslice_data)
+        write_to_pipe(:stats => hash_from_metric_data(unsent_timeslice_data))
+        {}
+      end
+
+      def transaction_sample_data(transactions)
+        write_to_pipe(:transaction_traces => transactions) if transactions
+      end
+
+      def error_data(errors)
+        write_to_pipe(:error_traces => errors) if errors
+      end
+
+      def sql_trace_data(sql)
+        write_to_pipe(:sql_traces => sql) if sql
+      end
+      
+      def shutdown(time)
+        write_to_pipe('EOF')
+        NewRelic::Agent::PipeChannelManager.channels[@channel_id].close
+      end
+      
+      private
+
+      def hash_from_metric_data(metric_data)
+        metric_hash = {}
+        metric_data.each do |metric_entry|
+          metric_hash[metric_entry.metric_spec] = metric_entry
+        end
+        metric_hash
+      end
+
+      def write_to_pipe(data)
+        NewRelic::Agent::PipeChannelManager.channels[@channel_id].write(data)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/samplers/delayed_job_sampler.rb

@@ -0,0 +1,89 @@
+require 'new_relic/agent/sampler'
+require 'new_relic/delayed_job_injection'
+
+module NewRelic
+  module Agent
+    module Samplers
+      # This sampler records the status of your delayed job table once a minute.
+      # It assumes jobs are cleared after being run, and failed jobs are not (otherwise
+      # the failed job metric is useless).
+      #
+      # In earlier versions it will break out the queue length by priority.  In later
+      # versions of DJ where distinct queues are supported, it breaks it out by queue name.
+      #
+      class DelayedJobSampler < NewRelic::Agent::Sampler
+        def initialize
+          super :delayed_job_queue
+          raise Unsupported, "DJ instrumentation disabled" if NewRelic::Control.instance['disable_dj']
+          raise Unsupported, "No DJ worker present" unless NewRelic::DelayedJobInjection.worker_name
+        end
+
+        def error_stats
+          stats_engine.get_stats("Workers/DelayedJob/failed_jobs", false)
+        end
+        def locked_job_stats
+          stats_engine.get_stats("Workers/DelayedJob/locked_jobs", false)
+        end
+
+        def local_env
+          NewRelic::Control.instance.local_env
+        end
+
+        def worker_name
+          local_env.dispatcher_instance_id
+        end
+
+        def queued_jobs
+          Delayed::Job.count(:conditions => ['run_at < ? and failed_at is NULL', Time.now])
+        end
+        def failed_jobs
+          Delayed::Job.count(:conditions => 'failed_at is not NULL')
+        end
+        def locked_jobs
+          Delayed::Job.count(:conditions => 'locked_by is not NULL')
+        end
+
+        def self.supported_on_this_platform?
+          defined?(Delayed::Job)
+        end
+
+        def poll
+          record error_stats, failed_jobs
+          record locked_job_stats, locked_jobs
+
+          if @queue
+            record_queue_length_across_dimension('queue')
+          else
+            record_queue_length_across_dimension('priority')
+          end
+        end
+
+        private
+
+        def record_queue_length_across_dimension(column)
+          all_count = 0
+          Delayed::Job.count(:group => column).each do | column_val, count |
+            all_count += count
+            record stats_engine.get_stats("Workers/DelayedJob/queue_length/#{column == 'queue' ? 'name' : column}/#{column_val}", false), count
+          end
+          record(stats_engine.get_stats("Workers/DelayedJob/queue_length/all", false), all_count)
+        end
+
+        # Figure out if we get the queues.
+        def setup
+          return unless @queue.nil?
+          @setup = true
+          columns = Delayed::Job.columns
+          columns.each do | c |
+            @queue = true if c.name.to_s == 'priority'
+          end
+          @queue ||= false
+        end
+
+        def record(stat, size)
+          stat.record_data_point size
+        end
+      end
+    end
+  end
+end