newrelic_rpm 3.4.0.beta1 → 3.4.0.beta2

data/CHANGELOG

@@ -1,15 +1,17 @@
 v3.4.0
  * Major refactor of data transmission mechanism.  This enabled child processes to send data to parent processes, which then send the data to the New Relic service.  This should only affect Resque users, dramatically improving their experience.
  * Moved Resque instrumentation from rpm_contrib to main agent.  Resque users should discontinue use of rpm_contrib or upgrade to 2.1.11.
+ * Resolve issue with configuring the Error Collector when using server-side configuration.
 
 v3.3.5
- * [FIX] Allow tracing of ! and ? methods
+ * [FIX] Allow tracing of methods ending in ! and ?
  * [PERF] Give up after scanning first 50k of the response in RUM
    auto-instrumentation.
- * [FIX] Don't raise when extracting metrics from SQL queries with non UTF-8
- * bytes.  Replaced "Custom/DJ Locked Jobs" metric with three new metrics for
+ * [FIX] Don't raise when extracting metrics from SQL queries with non UTF-8 bytes.
+ * Replaced "Custom/DJ Locked Jobs" metric with new metrics for
    monitoring DelayedJob: queue_length, failed_jobs, and locked_jobs, all under
-   Workers/DelayedJob
+   Workers/DelayedJob.  queue_length is also broken out by queue name or priority
+   depending on the version of DelayedJob deployed.
 
 v3.3.4.1
  * Bug fix when rendering empty collection in Rails 3.1+

data/lib/new_relic/agent/agent.rb

@@ -4,7 +4,6 @@ require 'net/http'
 require 'logger'
 require 'zlib'
 require 'stringio'
-require 'new_relic/data_serialization'
 require 'new_relic/agent/new_relic_service'
 require 'new_relic/agent/pipe_service'
 
@@ -155,6 +154,7 @@ module NewRelic
           if channel_id = options[:report_to_channel]
             @service = NewRelic::Agent::PipeService.new(channel_id)
             @connected_pid = $$
+            @metric_ids = {}
           end
           
           # log.debug "Agent received after_fork notice in #$$: [#{control.agent_enabled?}; monitor=#{control.monitor_mode?}; connected: #{@connected.inspect}; thread=#{@worker_thread.inspect}]"
@@ -444,9 +444,6 @@ module NewRelic
         end
 
         private
-#         def collector
-#           @collector ||= control.collector
-#         end
         
         # All of this module used to be contained in the
         # start_worker_thread method - this is an artifact of
@@ -485,7 +482,7 @@ module NewRelic
           def create_and_run_worker_loop
             @worker_loop = WorkerLoop.new
             @worker_loop.run(@report_period) do
-              save_or_transmit_data
+              transmit_data
             end
           end
 
@@ -869,7 +866,7 @@ module NewRelic
             config_transaction_tracer
             log_connection!(config_data)
             configure_transaction_tracer!(config_data['collect_traces'], config_data['sample_rate'])
-            configure_error_collector!(config_data['collect_errors'])
+            configure_error_collector!(config_data['error_collector.enabled'])
           end
           
           # Logs when we connect to the server, for debugging purposes
@@ -1009,8 +1006,6 @@ module NewRelic
         # transmission later
         def harvest_and_send_timeslice_data
           now = Time.now
-          NewRelic::Agent.instance.stats_engine.get_stats_no_scope('Supportability/invoke_remote').record_data_point(0.0)
-          NewRelic::Agent.instance.stats_engine.get_stats_no_scope('Supportability/invoke_remote/metric_data').record_data_point(0.0)
           harvest_timeslice_data(now)
           # In this version of the protocol, we get back an assoc array of spec to id.            
           metric_specs_and_ids = @service.metric_data(@last_harvest_time.to_f,
@@ -1112,21 +1107,13 @@ module NewRelic
           end
         end
         
-        def save_or_transmit_data
-          if NewRelic::DataSerialization.should_send_data?
-            log.debug "Sending data to New Relic Service"
-            NewRelic::Agent.load_data unless NewRelic::Control.instance.disable_serialization?
-            harvest_and_send_errors
-            harvest_and_send_slowest_sample
-            harvest_and_send_slowest_sql
-            harvest_and_send_timeslice_data
-          else
-            log.debug "Serializing agent data to disk"
-            NewRelic::Agent.save_data
-          end
+        def transmit_data
+          log.debug "Sending data to New Relic Service"
+          harvest_and_send_errors
+          harvest_and_send_slowest_sample
+          harvest_and_send_slowest_sql
+          harvest_and_send_timeslice_data
         rescue => e
-          NewRelic::Control.instance.disable_serialization = true
-          NewRelic::Control.instance.log.warn("Disabling serialization: #{e.message}")
           retry_count ||= 0
           retry_count += 1
           retry unless retry_count > 1
@@ -1146,7 +1133,7 @@ module NewRelic
           if @connected
             begin
               @service.request_timeout = 10
-              save_or_transmit_data
+              transmit_data
               if @connected_pid == $$ && !@service.kind_of?(NewRelic::Agent::NewRelicService)
                 log.debug "Sending New Relic service agent run shutdown message"
                 @service.shutdown(Time.now.to_f)

data/lib/new_relic/agent/pipe_channel_manager.rb

@@ -118,7 +118,7 @@ module NewRelic
           got = pipe.read
           
           if got && !got.empty?
-            payload = Marshal.load(got)
+            payload = unmarshal(got)
             if payload == 'EOF'
               pipe.close
             else
@@ -128,6 +128,16 @@ module NewRelic
             end
           end
         end
+
+        def unmarshal(data)
+          if NewRelic::LanguageSupport.broken_gc?
+            NewRelic::LanguageSupport.with_disabled_gc do
+              Marshal.load(data)
+            end
+          else
+            Marshal.load(data)
+          end
+        end
         
         def should_keep_listening?
           @started || @pipes.values.find{|pipe| !pipe.in.closed?}

data/lib/new_relic/agent/pipe_service.rb

@@ -1,14 +1,13 @@
 module NewRelic
   module Agent
     class PipeService
-      attr_reader :channel_id, :buffer, :stats_engine
+      attr_reader :channel_id, :buffer
       attr_accessor :request_timeout, :agent_id, :collector
       
       def initialize(channel_id)
         @channel_id = channel_id
         @collector = NewRelic::Control::Server.new(:name => 'parent',
                                                    :port => 0)
-        @stats_engine = NewRelic::Agent::StatsEngine.new
       end
       
       def connect(config)
@@ -16,9 +15,7 @@ module NewRelic
       end
       
       def metric_data(last_harvest_time, now, unsent_timeslice_data)
-        @stats_engine.merge_data(hash_from_metric_data(unsent_timeslice_data))
-        stats = @stats_engine.harvest_timeslice_data({}, {})
-        write_to_pipe(:stats => stats) if stats
+        write_to_pipe(:stats => hash_from_metric_data(unsent_timeslice_data))
         {}
       end
 

data/lib/new_relic/agent/samplers/delayed_job_sampler.rb

@@ -4,6 +4,13 @@ require 'new_relic/delayed_job_injection'
 module NewRelic
   module Agent
     module Samplers
+      # This sampler records the status of your delayed job table once a minute.
+      # It assumes jobs are cleared after being run, and failed jobs are not (otherwise
+      # the failed job metric is useless).
+      #
+      # In earlier versions it will break out the queue length by priority.  In later
+      # versions of DJ where distinct queues are supported, it breaks it out by queue name.
+      #
       class DelayedJobSampler < NewRelic::Agent::Sampler
         def initialize
           super :delayed_job_queue
@@ -11,9 +18,6 @@ module NewRelic
           raise Unsupported, "No DJ worker present" unless NewRelic::DelayedJobInjection.worker_name
         end
 
-        def queue_stats
-          stats_engine.get_stats("Workers/DelayedJob/queue_length", false)
-        end
         def error_stats
           stats_engine.get_stats("Workers/DelayedJob/failed_jobs", false)
         end
@@ -44,12 +48,38 @@ module NewRelic
         end
 
         def poll
-          record queue_stats, queued_jobs
           record error_stats, failed_jobs
           record locked_job_stats, locked_jobs
+
+          if @queue
+            record_queue_length_across_dimension('queue')
+          else
+            record_queue_length_across_dimension('priority')
+          end
         end
 
         private
+
+        def record_queue_length_across_dimension(column)
+          all_count = 0
+          Delayed::Job.count(:group => column).each do | column_val, count |
+            all_count += count
+            record stats_engine.get_stats("Workers/DelayedJob/queue_length/#{column == 'queue' ? 'name' : column}/#{column_val}", false), count
+          end
+          record(stats_engine.get_stats("Workers/DelayedJob/queue_length/all", false), all_count)
+        end
+
+        # Figure out if we get the queues.
+        def setup
+          return unless @queue.nil?
+          @setup = true
+          columns = Delayed::Job.columns
+          columns.each do | c |
+            @queue = true if c.name.to_s == 'priority'
+          end
+          @queue ||= false
+        end
+
         def record(stat, size)
           stat.record_data_point size
         end

data/lib/new_relic/agent.rb

@@ -1,6 +1,5 @@
 require 'forwardable'
 require 'new_relic/control'
-require 'new_relic/data_serialization'
 
 # = New Relic Ruby Agent
 #
@@ -223,45 +222,6 @@ module NewRelic
     def shutdown(options={})
       agent.shutdown(options)
     end
-    
-    # a method used to serialize short-running processes to disk, so
-    # we don't incur the overhead of reporting to the server for every
-    # fork/invocation of a small job.
-    #
-    # Functionally, this loads the data from the file into the agent
-    # (to avoid losing data by overwriting) and then serializes the
-    # agent data to the file again. See also #load_data
-    def save_data
-      NewRelic::DataSerialization.read_and_write_to_file do |old_data|
-        agent.merge_data_from(old_data)
-        agent.serialize
-      end
-    end
-    
-    # used to load data from the disk during the harvest cycle to send
-    # it. This method also clears the file so data should never be
-    # sent more than once.
-
-    # Note that only one transaction trace will be sent even if many
-    # are serialized, since the slowest is sent.
-    #
-    # See also the complement to this method, #save_data - used when a
-    # process is shutting down
-    def load_data
-      if !NewRelic::Control.instance['disable_serialization']
-        NewRelic::DataSerialization.read_and_write_to_file do |old_data|
-          agent.merge_data_from(old_data)
-          nil # return nil so nothing is written to the file
-        end
-        NewRelic::DataSerialization.update_last_sent!
-      end
-      
-      {
-        :metrics => agent.stats_engine.metrics.length,
-        :traces => agent.unsent_traces_size,
-        :errors => agent.unsent_errors_size
-      }
-    end
 
     # Add instrumentation files to the agent.  The argument should be
     # a glob matching ruby scripts which will be executed at the time

data/lib/new_relic/language_support.rb

@@ -1,28 +1,6 @@
 module NewRelic::LanguageSupport
   extend self
   
-  module DataSerialization
-    def self.included(base)
-      # need to disable GC during marshal load in 1.8.7
-      if NewRelic::LanguageSupport.using_version?('1.8.7') &&
-          !NewRelic::LanguageSupport.using_engine?('jruby') &&
-          !NewRelic::LanguageSupport.using_engine?('rbx')
-        base.class_eval do
-          def self.load(*args)
-            if defined?(::GC) && ::GC.respond_to?(:disable)
-              ::GC.disable
-              val = super
-              ::GC.enable
-              val
-            else
-              super
-            end
-          end
-        end
-      end
-    end
-  end
-  
   module Control
     def self.included(base)
       # need to use syck rather than psych when possible
@@ -81,6 +59,24 @@ module NewRelic::LanguageSupport
       engine == 'ruby'
     end
   end
+
+  def broken_gc?
+    NewRelic::LanguageSupport.using_version?('1.8.7') &&
+      RUBY_PATCHLEVEL < 348 &&
+      !NewRelic::LanguageSupport.using_engine?('jruby') &&
+      !NewRelic::LanguageSupport.using_engine?('rbx')
+  end
+
+  def with_disabled_gc
+    if defined?(::GC) && ::GC.respond_to?(:disable)
+      ::GC.disable
+      val = yield
+      ::GC.enable
+      val
+    else
+      yield
+    end
+  end
   
   def using_version?(version)
     numbers = version.split('.')

data/lib/new_relic/version.rb

@@ -4,7 +4,7 @@ module NewRelic
     MAJOR = 3
     MINOR = 4
     TINY  = 0
-    BUILD = 'beta1' # Set to nil for a release, 'beta1', 'alpha', etc for prerelease builds
+    BUILD = 'beta2' # Set to nil for a release, 'beta1', 'alpha', etc for prerelease builds
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end
 

data/newrelic_rpm.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "newrelic_rpm"
-  s.version = "3.4.0.beta1"
+  s.version = "3.4.0.beta2"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["Bill Kayser", "Jon Guymon", "Justin George", "Darin Swanson"]
-  s.date = "2012-05-25"
+  s.date = "2012-06-07"
   s.description = "New Relic is a performance management system, developed by New Relic,\nInc (http://www.newrelic.com).  New Relic provides you with deep\ninformation about the performance of your web application as it runs\nin production. The New Relic Ruby Agent is dual-purposed as a either a\nGem or plugin, hosted on\nhttp://github.com/newrelic/rpm/\n"
   s.email = "support@newrelic.com"
-  s.executables = ["newrelic", "mongrel_rpm", "newrelic_cmd"]
+  s.executables = ["newrelic_cmd", "newrelic", "mongrel_rpm"]
   s.extra_rdoc_files = [
     "CHANGELOG",
     "LICENSE",
@@ -105,7 +105,6 @@ Gem::Specification.new do |s|
     "lib/new_relic/control/logging_methods.rb",
     "lib/new_relic/control/profiling.rb",
     "lib/new_relic/control/server_methods.rb",
-    "lib/new_relic/data_serialization.rb",
     "lib/new_relic/delayed_job_injection.rb",
     "lib/new_relic/helper.rb",
     "lib/new_relic/language_support.rb",
@@ -189,7 +188,6 @@ Gem::Specification.new do |s|
     "test/new_relic/control/configuration_test.rb",
     "test/new_relic/control/logging_methods_test.rb",
     "test/new_relic/control_test.rb",
-    "test/new_relic/data_serialization_test.rb",
     "test/new_relic/delayed_job_injection_test.rb",
     "test/new_relic/fake_collector.rb",
     "test/new_relic/fake_service.rb",

data/test/new_relic/agent/agent/connect_test.rb

@@ -363,7 +363,7 @@ class NewRelic::Agent::Agent::ConnectTest < Test::Unit::TestCase
       'data_report_period' => 'pasta sauce',
       'url_rules' => 'tamales',
       'collect_traces' => true,
-      'collect_errors' => true,
+      'error_collector.enabled' => true,
       'sample_rate' => 10
     }
     NewRelic::Control.instance.settings['transaction_tracer'] = {'enabled' => true}

data/test/new_relic/agent/agent/start_worker_thread_test.rb

@@ -64,7 +64,7 @@ class NewRelic::Agent::Agent::StartWorkerThreadTest < Test::Unit::TestCase
     wl = mock('worker loop')
     NewRelic::Agent::WorkerLoop.expects(:new).returns(wl)
     wl.expects(:run).with(30).yields
-    self.expects(:save_or_transmit_data)
+    self.expects(:transmit_data)
     create_and_run_worker_loop
   end
 

data/test/new_relic/agent/agent_test.rb

@@ -5,6 +5,7 @@ module NewRelic
       def setup
         super
         @agent = NewRelic::Agent::Agent.new
+        @agent.service = NewRelic::FakeService.new
       end
 
       def test_after_fork_reporting_to_channel
@@ -13,37 +14,21 @@ module NewRelic
                'Agent should use PipeService when directed to report to pipe channel')
         assert_equal 123, @agent.service.channel_id
       end
-
-      def test_save_or_transmit_data_should_save
-        NewRelic::Agent.expects(:save_data).once
-        @agent.expects(:harvest_and_send_timeslice_data).never
-        NewRelic::DataSerialization.expects(:should_send_data?).returns(false)
-        @agent.instance_eval { save_or_transmit_data }
-      end
       
-      def test_save_or_transmit_data_should_transmit
-        NewRelic::Control.instance.stubs(:disable_serialization?).returns(false)
-        NewRelic::Agent.expects(:load_data)
-        @agent.expects(:harvest_and_send_timeslice_data)
-        @agent.expects(:harvest_and_send_slowest_sample)
-        @agent.expects(:harvest_and_send_errors)
-        NewRelic::DataSerialization.expects(:should_send_data?).returns(true)
-        @agent.instance_eval { save_or_transmit_data }
+      def test_transmit_data_should_transmit
+        @agent.instance_eval { transmit_data }
+        assert @agent.service.agent_data.any?
       end
 
-      def test_save_or_transmit_data_should_close_explain_db_connections
-        NewRelic::Agent.stubs(:save_data)
-        NewRelic::DataSerialization.expects(:should_send_data?).returns(false)
+      def test_transmit_data_should_close_explain_db_connections
         NewRelic::Agent::Database.expects(:close_connections)
-        @agent.instance_eval { save_or_transmit_data }
+        @agent.instance_eval { transmit_data }
       end
       
-      def test_save_or_transmit_data_should_not_close_db_connections_if_forked
-        NewRelic::Agent.stubs(:save_data)
-        NewRelic::DataSerialization.expects(:should_send_data?).returns(false)
+      def test_transmit_data_should_not_close_db_connections_if_forked
         NewRelic::Agent::Database.expects(:close_connections).never
         @agent.after_fork
-        @agent.instance_eval { save_or_transmit_data }
+        @agent.instance_eval { transmit_data }
       end
 
       def test_serialize

data/test/new_relic/agent/method_tracer_test.rb

@@ -244,6 +244,7 @@ class NewRelic::Agent::MethodTracerTest < Test::Unit::TestCase
     t1 = Time.now
     method_with_block(1,2,3,true,METRIC) do |scope|
       assert scope == METRIC
+      sleep 0.1 # pad the test a bit to increase the margin of error
     end
     elapsed = Time.now - t1
 

data/test/new_relic/agent_test.rb

@@ -49,6 +49,10 @@ module NewRelic
         NewRelic::Agent.instance.stats_engine.get_stats_no_scope(metric) \
           .record_data_point(1.0)
         
+        # ensure that cached metric ids don't interfere with metric merging
+        NewRelic::Agent.agent.instance_variable_set(:@metric_ids,
+                                    {NewRelic::MetricSpec.new('Instance/Busy') => 1})
+        
         NewRelic::Agent::PipeChannelManager.listener.close_all_pipes
         NewRelic::Agent.register_report_channel(:agent_test) # before fork
         pid = Process.fork do
@@ -201,25 +205,6 @@ module NewRelic
     def test_instance
       assert_equal(NewRelic::Agent.agent, NewRelic::Agent.instance, "should return the same agent for both identical methods")
     end
-
-    def test_load_data_should_disable_serialization_if_an_error_is_encountered
-      NewRelic::Control.instance['disable_serialization'] = false
-      NewRelic::DataSerialization.stubs(:should_send_data?).returns(false)
-      NewRelic::Agent.stubs(:save_data).raises(Errno::EACCES)
-      begin
-        NewRelic::Agent.instance.send(:save_or_transmit_data)
-      rescue Errno::EACCES; end
-      # should be true
-      assert(NewRelic::Control.instance['disable_serialization'])
-      NewRelic::Control.instance['disable_serialization'] = false
-    end
-
-    def test_load_data_should_not_write_files_when_serialization_disabled
-      NewRelic::Control.instance['disable_serialization'] = true
-      NewRelic::DataSerialization.expects(:read_and_write_to_file).never
-      NewRelic::Agent.load_data
-      NewRelic::Control.instance['disable_serialization'] = false
-    end
     
     def test_register_report_channel
       NewRelic::Agent.register_report_channel(:channel_id)

data/test/new_relic/fake_service.rb

@@ -34,7 +34,7 @@ module NewRelic
     
     def method_missing(method, *args)
       if @supported_methods.include?(method)
-        @agent_data << OpenStruct.new(:method => method, :params => args)
+        @agent_data << OpenStruct.new(:action => method, :params => args)
         @mock[method.to_s]
       else
         super

data/test/script/ci.sh

@@ -51,9 +51,10 @@ mkdir -p tmp
 cd tmp
 
 
-rpm_test_app_cache=~/.rpm_test_app_cache
+#rpm_test_app_cache=~/.rpm_test_app_cache
+rpm_test_app_cache=~/workspace/.rpm_test_app_cache
 (
-  echo "updating local cache of rpm_test_app"
+  echo "updating local cache of rpm_test_app in $rpm_test_app_cache"
   git clone --mirror git://github.com/newrelic/rpm_test_app.git $rpm_test_app_cache || true
   cd $rpm_test_app_cache
 )
@@ -127,7 +128,3 @@ fi
 export RAILS_ENV=test
 bundle
 bundle exec rake --trace db:create:all ci:setup:testunit test:newrelic
-
-
-
-

data/test/script/ci_multiverse_runner.sh

@@ -1,11 +1,28 @@
 #!/bin/bash -e
 
+export PATH=$PATH:$HOME/bin
+
+echo $HOME
+echo $PATH
+#ls $HOME/bin
+
+
+
+if [ "x$RUBY" == "x" ] ; then
+	export RUBY=1.9.3
+fi
+
+echo "Tests will be run using $RUBY"
+#uname -a
+
 SCRATH_DIR=./multiverse_tmp
 script_dirname=`dirname $0`
 
 # make sure that we're in the project root
 cd "$script_dirname/../../"
 
+#pwd 
+
 if [ -x $SCRATH_DIR ] ; then
 	echo "found tmp, deleting"
 	rm -fr $SCRATH_DIR
@@ -14,22 +31,33 @@ fi
 mkdir $SCRATH_DIR
 cd $SCRATH_DIR
 
+#pwd 
+if [[ $JOB_NAME =~ "Pangalactic" ]] ; then 
+	AGENT_LOCATION="../../../../../../Ruby_Agent"
+else
+	AGENT_LOCATION="../../Ruby_Agent"
+fi
+
 git clone --depth=1 git@github.com:newrelic/multiverse.git multiverse
 git clone --depth=1 git@github.com:newrelic/rpm_contrib.git rpm_contrib
 
-if [ -x ../../Ruby_Agent ] ; then
-	ln -s ../../Ruby_Agent ./ruby_agent
+echo "Looking for Ruby Agent at $AGENT_LOCATION"
+#ls -l ../../../../../../
+#ls -l /home/hudson/workspace/
+
+if [ -x $AGENT_LOCATION ] ; then
+	ln -s $AGENT_LOCATION ./ruby_agent
 else
-	echo "*********** Ruby_Agent can't be found ***********"
+	echo "*********** Ruby_Agent not found ***********"
 	exit 1
 fi
 
 cd multiverse
 #./ci_run.sh
 
-pwd
-ls -l ../
+#pwd
+#ls -l ../
 
 source ~/.rvm/scripts/rvm
-rvm use 1.9.3
+rvm use $RUBY
 script/runner

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: newrelic_rpm
 version: !ruby/object:Gem::Version 
-  hash: 1309669669
+  hash: 62196359
   prerelease: 6
   segments: 
   - 3
   - 4
   - 0
   - beta
-  - 1
-  version: 3.4.0.beta1
+  - 2
+  version: 3.4.0.beta2
 platform: ruby
 authors: 
 - Bill Kayser
@@ -20,7 +20,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-25 00:00:00 Z
+date: 2012-06-07 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: jeweler
@@ -74,9 +74,9 @@ description: |
 
 email: support@newrelic.com
 executables: 
+- newrelic_cmd
 - newrelic
 - mongrel_rpm
-- newrelic_cmd
 extensions: []
 
 extra_rdoc_files: 
@@ -170,7 +170,6 @@ files:
 - lib/new_relic/control/logging_methods.rb
 - lib/new_relic/control/profiling.rb
 - lib/new_relic/control/server_methods.rb
-- lib/new_relic/data_serialization.rb
 - lib/new_relic/delayed_job_injection.rb
 - lib/new_relic/helper.rb
 - lib/new_relic/language_support.rb
@@ -254,7 +253,6 @@ files:
 - test/new_relic/control/configuration_test.rb
 - test/new_relic/control/logging_methods_test.rb
 - test/new_relic/control_test.rb
-- test/new_relic/data_serialization_test.rb
 - test/new_relic/delayed_job_injection_test.rb
 - test/new_relic/fake_collector.rb
 - test/new_relic/fake_service.rb
@@ -421,3 +419,4 @@ specification_version: 3
 summary: New Relic Ruby Agent
 test_files: []
 
+has_rdoc: 

data/lib/new_relic/data_serialization.rb

@@ -1,151 +0,0 @@
-require 'fileutils'
-require 'new_relic/language_support'
-
-module NewRelic
-  # Handles serialization of data to disk, to save on contacting the
-  # server. Lowers both server and client overhead, if the disk is not overloaded
-  class DataSerialization
-    include NewRelic::LanguageSupport::DataSerialization
-    
-    module ClassMethods
-      # Check whether the store is too large, too old, or the
-      # pid file is too old. If so, we should send the data
-      # right away. If not, we presumably store it for later sending
-      # (handled elsewhere)
-      def should_send_data?
-        NewRelic::Control.instance.disable_serialization? || store_too_large? ||
-          store_too_old? || pid_too_old? ||
-          NewRelic::LanguageSupport.using_version?('1.8.6')
-      rescue => e
-        NewRelic::Control.instance.disable_serialization = true
-        NewRelic::Control.instance.log.warn("Disabling serialization: #{e.message}")
-        true
-      end
-
-      # A combined locked read/write from the store file - reduces
-      # contention by not acquiring the lock and file handle twice
-      def read_and_write_to_file
-        with_locked_store do |f|
-          result = (yield get_data_from_file(f))
-          f.rewind
-          f.truncate(0)
-          write_contents_nonblockingly(f, dump(result)) if result
-        end
-      rescue Errno::ENOENT => e
-        NewRelic::Control.instance.log.warn(e.message)
-      end
-      
-      # touches the age file that determines whether we should send
-      # data now or not
-      def update_last_sent!
-        FileUtils.touch(pid_file_path)
-      end
-      
-      def pid_too_old?
-        return true unless File.exists?(pid_file_path)
-        age = (Time.now.to_i - File.mtime(pid_file_path).to_i)
-        NewRelic::Control.instance.log.debug("Pid was #{age} seconds old, sending data") if age > 60
-        age > 60
-      end
-      
-      def store_too_old?
-        return true unless File.exists?(file_path)
-        age = (Time.now.to_i - File.mtime(file_path).to_i)
-        NewRelic::Control.instance.log.debug("Store was #{age} seconds old, sending data") if age > 60
-        age > 50
-      end      
-    
-      def store_too_large?
-        return true unless File.exists?(file_path)
-        size = File.size(file_path) > max_size
-        NewRelic::Control.instance.log.debug("Store was oversize, sending data") if size
-        size
-      end
-      
-      private
-      
-      def open_arguments
-        if defined?(Encoding)
-          [file_path, File::RDWR | File::CREAT, {:internal_encoding => nil}]
-        else
-          [file_path, File::RDWR | File::CREAT]
-        end
-      end
-      
-      def with_locked_store
-        File.open(*open_arguments) do |f|
-          f.flock(File::LOCK_EX)
-          begin
-            yield(f)
-          ensure
-            f.flock(File::LOCK_UN)
-          end
-        end
-      rescue => e
-        NewRelic::Control.instance.log.error("Error serializing data to disk: #{e.inspect}")
-        NewRelic::Control.instance.log.debug(e.backtrace.split("\n"))
-        # re-raise so that serialization will be disabled higher up the stack
-        raise e
-      end
-
-      def get_data_from_file(f)
-        data = read_until_eof_error(f)
-        result = load(data)
-        f.truncate(0)
-        result
-      end
-
-      def write_contents_nonblockingly(f, string)
-        result = 0
-        while (result < string.length)
-          result += f.write_nonblock(string)
-        end
-      rescue Errno::EAGAIN, Errno::EINTR
-        IO.select(nil, [f])
-        retry
-      end
-
-      def read_until_eof_error(f)
-        accumulator = ""
-        while(true)
-          accumulator << f.read_nonblock(10_000)
-        end
-      rescue Errno::EAGAIN, Errno::EINTR
-        IO.select([f])
-        retry
-      rescue EOFError
-        accumulator
-      end
-      
-      def max_size
-        10_000
-      end
-
-      def dump(object)
-        Marshal.dump(object.clone)
-      end
-      
-      def load(dump)
-        if dump.respond_to?(:size) && dump.size == 0
-          NewRelic::Control.instance.log.debug("Spool file empty.")
-          return nil
-        end
-        Marshal.load(dump)
-      rescue ArgumentError, TypeError => e
-        NewRelic::Control.instance.log.error("Error loading data from newrelic_agent_store.db: #{e.inspect}")
-        NewRelic::Control.instance.log.debug(e.backtrace.inspect)
-        nil
-      end
-            
-      def file_path
-        "#{NewRelic::Control.instance.log_path}/newrelic_agent_store.db"
-      end
-
-      def pid_file_path
-        "#{NewRelic::Control.instance.log_path}/newrelic_agent_store.pid"
-      end
-    end
-    extend ClassMethods
-  end
-end
-

data/test/new_relic/data_serialization_test.rb

@@ -1,208 +0,0 @@
-require File.expand_path(File.join(File.dirname(__FILE__),'..', 'test_helper'))
-require 'new_relic/data_serialization'
-class NewRelic::DataSerializationTest < Test::Unit::TestCase
-
-  attr_reader :file, :path
-  
-  def setup
-    NewRelic::Control.instance['log_file_path'] = './log'
-    @path = NewRelic::Control.instance.log_path
-    @file = "#{path}/newrelic_agent_store.db"
-    Dir.mkdir(path) if !File.directory?(path)
-    FileUtils.rm_rf(@file)
-    FileUtils.rm_rf("#{@path}/newrelic_agent_store.pid")
-  end
-  
-  def teardown
-    # this gets set to true in some tests
-    NewRelic::Control.instance['disable_serialization'] = false
-    mocha_teardown
-  end
-  
-  def test_read_and_write_from_file_read_only
-    File.open(file, 'w') do |f|
-      f.write(Marshal.dump('a happy string'))
-    end
-    NewRelic::DataSerialization.read_and_write_to_file do |data|
-      assert_equal('a happy string', data, "should pull the dumped item from the file")
-      nil # must explicitly return nil or the return value will be dumped
-    end
-    assert_equal(0, File.size(file), "Should not leave any data in the file")
-  end
-  
-  def test_read_and_write_to_file_dumping_contents
-    expected_contents = Marshal.dump('a happy string')
-    NewRelic::DataSerialization.read_and_write_to_file do
-      'a happy string'
-    end
-    assert_equal(expected_contents, File.read(file), "should have dumped the contents")
-  end
-
-  def test_read_and_write_to_file_yields_old_data
-    expected_contents = 'a happy string'
-    File.open(file, 'w') do |f|
-      f.write(Marshal.dump(expected_contents))
-    end
-    contents = nil
-    NewRelic::DataSerialization.read_and_write_to_file do |old_data|
-      contents = old_data
-      'a happy string'
-    end
-    assert_equal(contents, expected_contents, "should have dumped the contents")
-  end
-
-  def test_read_and_write_to_file_round_trip
-    old_data = nil
-    NewRelic::DataSerialization.read_and_write_to_file do |data|
-      old_data = data
-      'a' * 30
-    end
-    NewRelic::DataSerialization.read_and_write_to_file do |data|
-      assert_equal('a'*30, data, "should be the same after serialization")
-    end
-  end
-
-  def test_should_send_data_when_over_limit
-    NewRelic::DataSerialization.stubs(:max_size).returns(20)
-    NewRelic::DataSerialization.read_and_write_to_file do
-      "a" * 30
-    end
-    assert(NewRelic::DataSerialization.should_send_data?, 'Should be over limit')
-  end
-
-  def test_read_until_eoferror
-    File.open(file, 'w') do |f|
-      f.write("a" * 10_001)
-    end
-    value = ""
-    File.open(file,'r') do |f|
-      value << NewRelic::DataSerialization.instance_eval { read_until_eof_error(f) }
-    end
-    assert_equal('a' * 10_001, value, "should retrieve all the contents from the string and not raise EOFerrors")
-  end
-  
-  def test_write_contents_nonblockingly
-    File.open(file, 'w') do |f|
-      f.write("") # write nothing! NOTHING
-    end
-
-    File.open(file, 'w') do |f|
-      NewRelic::DataSerialization.instance_eval { write_contents_nonblockingly(f, 'a' * 10_001) }
-    end
-    value = File.read(file)
-    assert_equal('a' * 10_001, value, "should write a couple thousand 'a's to a file without exploding")
-  end
-
-  def test_should_send_data_disabled
-    NewRelic::Control.instance.disable_serialization = true
-    assert(NewRelic::DataSerialization.should_send_data?,
-           'should send data when disabled')
-  end
-
-  def test_should_send_data_under_limit
-    NewRelic::DataSerialization.expects(:max_size).returns(2000)
-    NewRelic::DataSerialization.read_and_write_to_file do |old_data|
-      "a" * 5
-    end
-    
-    assert(!NewRelic::DataSerialization.store_too_large?,
-           'Should be under the limit')
-  end
-
-  def test_should_handle_empty_spool_file
-    NewRelic::Control.instance.log.expects(:error).never
-    assert_nil NewRelic::DataSerialization.instance_eval { load('') }
-  end
-
-  def test_spool_file_location_respects_log_file_path_setting
-    NewRelic::Control.instance.expects(:log_path).returns('./tmp')
-    Dir.mkdir('./tmp') if !File.directory?('./tmp')
-    NewRelic::DataSerialization.read_and_write_to_file do |_|
-      'a' * 30
-    end
-    assert(File.exists?('./tmp/newrelic_agent_store.db'),
-           "Spool file not created at user specified location")
-  end
-
-  def test_age_file_location_respects_log_file_path_setting
-    NewRelic::Control.instance.expects(:log_path).returns('./tmp')
-    Dir.mkdir('./tmp') if !File.directory?('./tmp')
-    NewRelic::DataSerialization.update_last_sent!
-    assert(File.exists?('./tmp/newrelic_agent_store.pid'),
-           "Age file not created at user specified location")
-  end
-  
-  def test_pid_age_creates_pid_file_if_none_exists
-    assert(!File.exists?("#{@path}/newrelic_agent_store.pid"),
-           'pid file found, should not be there')
-    NewRelic::DataSerialization.update_last_sent!
-    assert(File.exists?("#{@path}/newrelic_agent_store.pid"),
-           'pid file not found, should be there')
-  end
-  
-  def test_should_not_create_files_if_serialization_disabled
-    NewRelic::Control.instance['disable_serialization'] = true
-    NewRelic::DataSerialization.should_send_data?
-    assert(!File.exists?("#{@path}/newrelic_agent_store.db"),
-           'db file created when serialization disabled')
-    assert(!File.exists?("#{@path}/newrelic_agent_store.pid"),
-           'pid file created when serialization disabled')
-  end
-  
-  def test_loading_does_not_seg_fault_if_gc_triggers
-    return if NewRelic::LanguageSupport.using_version?('1.8.6')
-    require 'timeout'
-    
-    Thread.abort_on_exception = true
-    rcv,snd = IO.pipe
-    
-    write = Thread.new do
-      obj = ('a'..'z').inject({}){|h,s|h[s.intern]=s*1024;h}
-      data = Marshal.dump(obj)
-      snd.write(data[0,data.size/2])
-      sleep(0.1)
-      snd.write(data[(data.size/2)..-1])
-      snd.close
-    end
-    
-    read = Thread.new do
-      lock = Mutex.new
-      lock.synchronize do
-        NewRelic::DataSerialization.class_eval { load(rcv) }
-      end
-    end
-
-    gc = Thread.new do
-      10.times do
-        GC.start
-      end
-    end
-
-    Timeout::timeout(5) do
-      write.join
-      read.join
-      gc.join
-    end
-    # should not seg fault
-  end
-
-  def test_dump_should_be_thread_safe
-    stats_hash = {}
-
-    2000.times do |i|
-      stats_hash[i.to_s] = NewRelic::StatsBase.new
-    end
-        
-    harvest = Thread.new do
-      NewRelic::DataSerialization.class_eval { dump(stats_hash) }
-    end
-    
-    app = Thread.new do
-      stats_hash["a"] = NewRelic::StatsBase.new
-    end
-    
-    assert_nothing_raised do
-      [app, harvest].each{|t| t.join}
-    end    
-  end
-end