newrelic_rpm 3.6.7.152 → 3.6.7.159.beta

data/CHANGELOG

@@ -12,6 +12,22 @@
   All settings in newrelic.yml can now be configured via environment variables.
   See https://newrelic.com/docs/ruby/ruby-agent-configuration for full details.
 
+  * Additional locking option for Resque (3.6.7.159)
+
+  There have been reports of worker process deadlocks in Resque when using the
+  Ruby agent. An optional lock is now available to avoid those deadlocks. See
+  https://newrelic.com/docs/ruby/resque-instrumentation for more details.
+
+  * HTTP connection setup timeout (3.6.7.159)
+
+  HTTP initialization in the agent lacked an appropriate timeout,
+  leading to dropouts in reporting under certain network error conditions.
+
+  * Unnecessary requests from Resque jobs (3.6.7.159)
+
+  An issue causing Resque jobs to unnecessarily make requests against New Relic
+  servers was fixed.
+
   * Fix compatibility issues with excon and curb instrumentation
 
   This release of the agent fixes a warning seen under certain circumstances

data/lib/new_relic/agent/agent.rb

@@ -46,7 +46,7 @@ module NewRelic
         @transaction_sampler   = NewRelic::Agent::TransactionSampler.new
         @sql_sampler           = NewRelic::Agent::SqlSampler.new
         @thread_profiler       = NewRelic::Agent::Commands::ThreadProfiler.new
-        @agent_command_router  = NewRelic::Agent::Commands::AgentCommandRouter.new(@service, @thread_profiler)
+        @agent_command_router  = NewRelic::Agent::Commands::AgentCommandRouter.new(@thread_profiler)
         @cross_app_monitor     = NewRelic::Agent::CrossAppMonitor.new(@events)
         @error_collector       = NewRelic::Agent::ErrorCollector.new
         @transaction_rules     = NewRelic::Agent::RulesEngine.new
@@ -537,6 +537,7 @@ module NewRelic
         # might be holding locks for background thread that aren't there anymore.
         def reset_objects_with_locks
           @stats_engine = NewRelic::Agent::StatsEngine.new
+          reset_harvest_locks
         end
 
         def add_harvest_sampler(subclass)
@@ -556,6 +557,34 @@ module NewRelic
             ::NewRelic::Agent.logger.debug "Running worker loop"
           end
 
+          # Accessor for the harvest lock
+          def harvest_lock
+            return nil if @worker_loop.nil?
+            @worker_loop.lock
+          end
+
+          # Synchronize with the harvest loop. If the harvest thread has taken
+          # a lock (DNS lookups, backticks, agent-owned locks, etc), and we
+          # fork while locked, this can deadlock child processes. For more
+          # details, see https://github.com/resque/resque/issues/1101
+          def synchronize_with_harvest
+            if harvest_lock
+              harvest_lock.synchronize do
+                yield
+              end
+            else
+              yield
+            end
+          end
+
+          # Some forking cases (like Resque) end up with harvest lock from the
+          # parent process orphaned in the child. Let it go before we proceed.
+          def reset_harvest_locks
+            return if harvest_lock.nil?
+
+            harvest_lock.unlock if harvest_lock.locked?
+          end
+
           # Creates the worker loop and loads it with the instructions
           # it should run every @report_period seconds
           def create_and_run_worker_loop

data/lib/new_relic/agent/commands/agent_command_router.rb

@@ -14,24 +14,25 @@ module NewRelic
   module Agent
     module Commands
       class AgentCommandRouter
-        attr_reader :service, :handlers
-
-        def initialize(service, thread_profiler)
-          @service = service
+        attr_reader :handlers
 
+        def initialize(thread_profiler)
           @handlers    = Hash.new { |*| Proc.new { |cmd| self.unrecognized_agent_command(cmd) } }
-
           @handlers['start_profiler'] = Proc.new { |cmd| thread_profiler.handle_start_command(cmd) }
           @handlers['stop_profiler']  = Proc.new { |cmd| thread_profiler.handle_stop_command(cmd) }
         end
 
+        def new_relic_service
+          NewRelic::Agent.instance.service
+        end
+
         def handle_agent_commands
           results = invoke_commands(get_agent_commands)
-          service.agent_command_results(results) unless results.empty?
+          new_relic_service.agent_command_results(results) unless results.empty?
         end
 
         def get_agent_commands
-          commands = service.get_agent_commands
+          commands = new_relic_service.get_agent_commands
           NewRelic::Agent.logger.debug "Received get_agent_commands = #{commands.inspect}"
           commands
         end

data/lib/new_relic/agent/configuration/default_source.rb

@@ -393,6 +393,12 @@ module NewRelic
           :type => Boolean,
           :description => 'Enable or disable spawning of a background thread that listens for connections from child processes. Primarily used for Resque instrumentation.'
         },
+        :'resque.use_harvest_lock' => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :description => 'Enable or disable synchronizing Resque job forking with New Relic\'s harvest thread. Defaulted to false. This helps prevent Resque jobs from deadlocking, but pauses starting new jobs during harvest.'
+        },
         :data_report_period => {
           :default => 60,
           :public => false,

data/lib/new_relic/agent/instrumentation/resque.rb

@@ -64,6 +64,26 @@ DependencyDetection.defer do
     end
 
     if NewRelic::LanguageSupport.can_fork?
+      # Resque::Worker#fork isn't around in Resque 2.x
+      if NewRelic::VersionNumber.new(::Resque::VERSION) < NewRelic::VersionNumber.new("2.0.0")
+        ::Resque::Worker.class_eval do
+          if NewRelic::Agent.config[:'resque.use_harvest_lock']
+            ::NewRelic::Agent.logger.info 'Installing Resque harvest/fork synchronization'
+            def fork_with_newrelic(*args, &block)
+              NewRelic::Agent.instance.synchronize_with_harvest do
+                fork_without_newrelic(*args, &block)
+
+                # Reached in parent, not expected in the child since Resque
+                # uses the block form of fork
+              end
+            end
+
+            alias_method :fork_without_newrelic, :fork
+            alias_method :fork, :fork_with_newrelic
+          end
+        end
+      end
+
       ::Resque.before_first_fork do
         NewRelic::Agent.manual_start(:dispatcher   => :resque,
                                      :sync_startup => true,

data/lib/new_relic/agent/new_relic_service.rb

@@ -170,16 +170,21 @@ module NewRelic
 
         # Immediately open a TCP connection to the server and leave it open for
         # multiple requests.
-        ::NewRelic::Agent.logger.debug("Opening TCP connection to #{http.address}:#{http.port}")
-        http.start
         begin
+          t0 = Time.now
+          ::NewRelic::Agent.logger.debug("Opening TCP connection to #{http.address}:#{http.port}")
+          NewRelic::TimerLib.timeout(@request_timeout) { http.start }
           @shared_tcp_connection = http
           block.call
+        rescue Timeout::Error
+          elapsed = Time.now - t0
+          ::NewRelic::Agent.logger.warn "Timed out opening connection to #{http.address}:#{http.port} after #{elapsed} seconds. If this problem persists, please see http://status.newrelic.com"
+          raise
         ensure
           @shared_tcp_connection = nil
           # Close the TCP socket
           ::NewRelic::Agent.logger.debug("Closing TCP connection to #{http.address}:#{http.port}")
-          http.finish
+          http.finish if http.started?
         end
       end
 

data/test/new_relic/agent/agent_test.rb

@@ -13,7 +13,7 @@ module NewRelic
         super
         @agent = NewRelic::Agent::Agent.new
         @agent.service = default_service
-        @agent.agent_command_router.stubs(:service).returns(@agent.service)
+        @agent.agent_command_router.stubs(:new_relic_service).returns(@agent.service)
         @agent.stubs(:start_worker_thread)
       end
 
@@ -335,6 +335,32 @@ module NewRelic
         assert_match( /No application name configured/i, logmsg )
       end
 
+      def test_synchronize_with_harvest
+        lock = Mutex.new
+        @agent.stubs(:harvest_lock).returns(lock)
+        @agent.harvest_lock.lock
+
+        started = false
+        done = false
+
+        thread = Thread.new do
+          started = true
+          @agent.synchronize_with_harvest do
+            done = true
+          end
+        end
+
+        until started do
+          sleep(0.001)
+        end
+        assert !done
+
+        @agent.harvest_lock.unlock
+        thread.join
+
+        assert done
+      end
+
     end
 
 

data/test/new_relic/agent/commands/agent_command_router_test.rb

@@ -33,9 +33,10 @@ class AgentCommandRouterTest < Test::Unit::TestCase
 
   def setup
     @service = stub
+    NewRelic::Agent.agent.stubs(:service).returns(@service)
     @calls = []
 
-    @agent_commands = NewRelic::Agent::Commands::AgentCommandRouter.new(@service, nil)
+    @agent_commands = NewRelic::Agent::Commands::AgentCommandRouter.new(nil)
     @agent_commands.handlers["bazzle"] = Proc.new { |args| handle_bazzle_command(args) }
     @agent_commands.handlers["boom"]   = Proc.new { |args| handle_boom_command(args) }
   end

data/test/new_relic/agent/new_relic_service_test.rb

@@ -15,7 +15,25 @@ class NewRelicServiceKeepAliveTest < Test::Unit::TestCase
   end
 
   def stub_net_http_handle(overrides = {})
-    stub('http_handle', :start => true, :finish => true, :address => '10.10.10.10', :port => 30303)
+    defaults = { :start => true, :finish => true, :address => '10.10.10.10', :port => 30303, :started? => true }
+    stub('http_handle', defaults.merge(overrides))
+  end
+
+  def test_session_handles_timeouts_opening_connection_gracefully
+    conn = stub_net_http_handle(:started? => false)
+    conn.stubs(:start).raises(Timeout::Error)
+    conn.stubs(:finish).raises(RuntimeError)
+    @service.stubs(:create_http_connection).returns(conn)
+
+    block_ran = false
+
+    assert_raises(Timeout::Error) do
+      @service.session do
+        block_ran = true
+      end
+    end
+
+    assert(!block_ran, "Expected block passed to #session to have not run")
   end
 
   def test_session_block_reuses_http_handle

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: newrelic_rpm
 version: !ruby/object:Gem::Version
-  version: 3.6.7.152
-  prerelease: 
+  version: 3.6.7.159.beta
+  prerelease: 10
 platform: ruby
 authors:
 - Jason Clark
@@ -41,7 +41,7 @@ cert_chain:
   cHUySWFQWE92bTNUOEc0TzZxWnZobkxoL1VpZW4rK0RqOGVGQmVjVFBvTThw
   VmpLM3BoNQpuL0V3dVpDY0U2Z2h0Q0NNCi0tLS0tRU5EIENFUlRJRklDQVRF
   LS0tLS0K
-date: 2013-09-05 00:00:00.000000000 Z
+date: 2013-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -910,21 +910,28 @@ post_install_message: ! "# New Relic Ruby Agent Release Notes #\n\n## v3.6.7 ##\
   recognized by the\n  Ruby agent. The agent now starts correctly in those worker
   processes.\n\n* Environment-based configuration\n\n  All settings in newrelic.yml
   can now be configured via environment variables.\n  See https://newrelic.com/docs/ruby/ruby-agent-configuration
-  for full details.\n\n* Fix compatibility issues with excon and curb instrumentation\n\n
-  \ This release of the agent fixes a warning seen under certain circumstances\n  with
-  the excon gem (most notably, when excon was used by fog), as well as\n  a bug with
-  the curb instrumentation that conflicted with the  feedzirra gem.\n\n* Allow license
-  key to be set by Capistrano variables\n\n  A license key can be passed via a Capistrano
-  variable where previously it\n  could only be in newrelic.yml. Thanks Chris Marshall
-  for the contribution!\n\n* Make HTTP client instrumentation aware of \"Host\" request
-  header\n\n  If a \"Host\" header is set explicitly on an HTTP request, that hostname
-  will\n  be used for external metrics. Thanks Mislav Marohnić for the contribution!\n\n*
-  Fix ActiveSupport::Concern warnings with MethodTracer\n\n  Including NewRelic::Agent::MethodTracer
-  in a class using Concerns could cause\n  deprecation warnings. Thanks Mike Połtyn
-  for the contribution!\n\n* Fix Authlogic constant name\n\n  Code checking for the
-  Authlogic module was using in the wrong case. Thanks\n  Dharam Gollapudi for the
-  contribution!\n\nSee https://github.com/newrelic/rpm/blob/master/CHANGELOG for a
-  full list of\nchanges.\n"
+  for full details.\n\n* Additional locking option for Resque (3.6.7.159)\n\n  There
+  have been reports of worker process deadlocks in Resque when using the\n  Ruby agent.
+  An optional lock is now available to avoid those deadlocks. See\n  https://newrelic.com/docs/ruby/resque-instrumentation
+  for more details.\n\n* HTTP connection setup timeout (3.6.7.159)\n\n  HTTP initialization
+  in the agent lacked an appropriate timeout,\n  leading to dropouts in reporting
+  under certain network error conditions.\n\n* Unnecessary requests from Resque jobs
+  (3.6.7.159)\n\n  An issue causing Resque jobs to unnecessarily make requests against
+  New Relic\n  servers was fixed.\n\n* Fix compatibility issues with excon and curb
+  instrumentation\n\n  This release of the agent fixes a warning seen under certain
+  circumstances\n  with the excon gem (most notably, when excon was used by fog),
+  as well as\n  a bug with the curb instrumentation that conflicted with the  feedzirra
+  gem.\n\n* Allow license key to be set by Capistrano variables\n\n  A license key
+  can be passed via a Capistrano variable where previously it\n  could only be in
+  newrelic.yml. Thanks Chris Marshall for the contribution!\n\n* Make HTTP client
+  instrumentation aware of \"Host\" request header\n\n  If a \"Host\" header is set
+  explicitly on an HTTP request, that hostname will\n  be used for external metrics.
+  Thanks Mislav Marohnić for the contribution!\n\n* Fix ActiveSupport::Concern warnings
+  with MethodTracer\n\n  Including NewRelic::Agent::MethodTracer in a class using
+  Concerns could cause\n  deprecation warnings. Thanks Mike Połtyn for the contribution!\n\n*
+  Fix Authlogic constant name\n\n  Code checking for the Authlogic module was using
+  in the wrong case. Thanks\n  Dharam Gollapudi for the contribution!\n\nSee https://github.com/newrelic/rpm/blob/master/CHANGELOG
+  for a full list of\nchanges.\n"
 rdoc_options:
 - --line-numbers
 - --inline-source