newrelic_rpm 3.5.5.540.dev → 3.5.6.42.beta

data/.gitignore

@@ -1,5 +1,4 @@
 Gemfile.lock
-newrelic_rpm.gemspec
 .DS\_Store
 .svn/
 *~
@@ -19,3 +18,4 @@ tags
 /lerg/
 gems/newrelic_rpm.gemspec
 gems/newrelic_rpm*.tar.gz
+lib/new_relic/build.rb

data/CHANGELOG

@@ -3,6 +3,13 @@
 
 ## v3.5.5 ##
 
+  * Add thread profiling support
+
+    Thread profiling performs statistical sampling of backtraces of all threads
+    within your Ruby processes. This feature requires MRI >= 1.9.2, and is
+    controlled via the New Relic web UI. JRuby support (in 1.9.x compat mode) is
+    considered experimental, due to issues with JRuby's Thread#backtrace.
+
   * Add audit logging capability
 
     The agent can now log all of the data it sends to the New Relic servers to

data/Rakefile

@@ -5,7 +5,7 @@ require "#{File.dirname(__FILE__)}/lib/tasks/all.rb"
 
 task :default => :test
 
-task :test => [:gemspec, 'test:newrelic']
+task :test => ['test:newrelic']
 
 namespace :test do
   desc "Run all tests"
@@ -14,7 +14,7 @@ namespace :test do
   agent_home = File.expand_path(File.dirname(__FILE__))
 
   desc "Run functional test suite for newrelic"
-  task :multiverse, [:suite, :mode] => [:gemspec] do |t, args|
+  task :multiverse, [:suite, :mode] => [] do |t, args|
     args.with_defaults(:suite => "", :mode => "")
     if args.mode == "run_one"
       puts `#{agent_home}/test/multiverse/script/run_one #{args.suite}`
@@ -24,7 +24,7 @@ namespace :test do
   end
 
   desc "Test the multiverse testing framework by executing tests in test/multiverse/test. Get meta with it."
-  task 'multiverse:self', [:suite, :mode] => [:gemspec] do |t, args|
+  task 'multiverse:self', [:suite, :mode] => [] do |t, args|
     args.with_defaults(:suite => "", :mode => "")
     puts ("Testing the multiverse testing framework...")
     test_files = FileList['test/multiverse/test/*_test.rb']
@@ -43,20 +43,11 @@ namespace :test do
 
 end
 
-desc 'Generate gemspec [ build_number, stage ]'
-task :gemspec, [ :build_number, :stage ] do |t, args|
-  require 'erb'
-  version = NewRelic::VERSION::STRING.split('.')[0..2]
-  version << args.build_number.to_s if args.build_number
-  version << args.stage.to_s        if args.stage
-
-  version_string = version.join('.')
-  gem_version    = Gem::VERSION
-  date           = Time.now.strftime('%Y-%m-%d')
-  files          = `git ls-files`.split + ['newrelic_rpm.gemspec']
-
-  template = ERB.new(File.read('newrelic_rpm.gemspec.erb'))
-  File.open('newrelic_rpm.gemspec', 'w') do |gemspec|
-    gemspec.write(template.result(binding))
+desc 'Record build number and stage'
+task :record_build, [ :build_number, :stage ] do |t, args|
+  build_string = args.build_number
+  build_string << ".#{args.stage}" if args.stage
+  File.open("lib/new_relic/build.rb", "w") do |f|
+    f.write("module NewRelic; module VERSION; BUILD='#{build_string}'; end; end\n")
   end
 end

data/lib/new_relic/agent/agent.rb

@@ -24,11 +24,15 @@ module NewRelic
         @launch_time = Time.now
 
         @metric_ids = {}
+        @events = NewRelic::Agent::EventListener.new
         @stats_engine = NewRelic::Agent::StatsEngine.new
         @transaction_sampler = NewRelic::Agent::TransactionSampler.new
         @sql_sampler = NewRelic::Agent::SqlSampler.new
         @thread_profiler = NewRelic::Agent::ThreadProfiler.new
+        @cross_process_monitor = NewRelic::Agent::CrossProcessMonitor.new(@events)
         @error_collector = NewRelic::Agent::ErrorCollector.new
+
+        @connect_state = :pending
         @connect_attempts = 0
 
         @last_harvest_time = Time.now
@@ -94,6 +98,10 @@ module NewRelic
         attr_reader :cross_process_encoding_bytes
         # service for communicating with collector
         attr_accessor :service
+        # Global events dispatcher. This will provides our primary mechanism
+        # for agent-wide events, such as finishing configuration, error notification
+        # and request before/after from Rack.
+        attr_reader :events
 
 
         # Returns the length of the unsent errors array, if it exists,
@@ -175,19 +183,21 @@ module NewRelic
           @forked = true
           Agent.config.apply_config(NewRelic::Agent::Configuration::ManualSource.new(options), 1)
 
-          # @connected gets false after we fail to connect or have an error
-          # connecting.  @connected has nil if we haven't finished trying to connect.
-          # or we didn't attempt a connection because this is the master process
-
           if channel_id = options[:report_to_channel]
             @service = NewRelic::Agent::PipeService.new(channel_id)
-            @connected_pid = $$
-            @metric_ids = {}
+            if connected?
+              @connected_pid = $$
+              @metric_ids = {}
+            else
+              ::NewRelic::Agent.logger.debug("Child process #{$$} not reporting to non-connected parent.")
+              @service.shutdown(Time.now)
+              disconnect
+            end
           end
 
           return if !Agent.config[:agent_enabled] ||
             !Agent.config[:monitor_mode] ||
-            @connected == false ||
+            disconnected? ||
             @worker_thread && @worker_thread.alive?
 
           ::NewRelic::Agent.logger.debug "Starting the worker thread in #{$$} after forking."
@@ -210,12 +220,6 @@ module NewRelic
           @started
         end
 
-        # Return nil if not yet connected, true if successfully started
-        # and false if we failed to start.
-        def connected?
-          @connected
-        end
-
         # Attempt a graceful shutdown of the agent, running the worker
         # loop if it exists and is running.
         #
@@ -518,7 +522,7 @@ module NewRelic
             ::NewRelic::Agent.logger.debug error.message
             reset_stats
             @metric_ids = {}
-            @connected = nil
+            @connect_state = :pending
             sleep 30
           end
 
@@ -577,7 +581,7 @@ module NewRelic
                 # just exit the thread.  If it returns nil
                 # that means it didn't try to connect because we're in the master.
                 connect(connection_options)
-                if @connected
+                if connected?
                   log_worker_loop_start
                   create_and_run_worker_loop
                   # never reaches here unless there is a problem or
@@ -610,60 +614,41 @@ module NewRelic
         # method - all of its methods are used in that context, so it
         # can be refactored at will. It should be fully tested
         module Connect
-          # the frequency with which we should try to connect to the
-          # server at the moment.
-          attr_accessor :connect_retry_period
           # number of attempts we've made to contact the server
           attr_accessor :connect_attempts
 
           # Disconnect just sets connected to false, which prevents
           # the agent from trying to connect again
           def disconnect
-            @connected = false
+            @connect_state = :disconnected
             true
           end
 
-          # We've tried to connect if @connected is not nil, or if we
-          # are forcing reconnection (i.e. in the case of an
-          # after_fork with long running processes)
-          def tried_to_connect?(options)
-            !(@connected.nil? || options[:force_reconnect])
+          def connected?
+            @connect_state == :connected
           end
 
-          # We keep trying by default, but you can disable it with the
-          # :keep_retrying option set to false
-          def should_keep_retrying?(options)
-            @keep_retrying = (options[:keep_retrying].nil? || options[:keep_retrying])
+          def disconnected?
+            @connect_state == :disconnected
+          end
+
+          # Don't connect if we're already connected, or if we tried to connect
+          # and were rejected with prejudice because of a license issue, unless
+          # we're forced to by force_reconnect.
+          def should_connect?(force=false)
+            force || (!connected? && !disconnected?)
           end
 
           # Retry period is a minute for each failed attempt that
           # we've made. This should probably do some sort of sane TCP
           # backoff to prevent hammering the server, but a minute for
           # each attempt seems to work reasonably well.
-          def get_retry_period
-            return 600 if self.connect_attempts > 6
-            connect_attempts * 60
-          end
-
-          def increment_retry_period! #:nodoc:
-            self.connect_retry_period=(get_retry_period)
+          def connect_retry_period
+            [600, connect_attempts * 60].min
           end
 
-          # We should only retry when there has not been a more
-          # serious condition that would prevent it. We increment the
-          # connect attempts and the retry period, to prevent constant
-          # connection attempts, and tell the user what we're doing by
-          # logging.
-          def should_retry?
-            if @keep_retrying
-              self.connect_attempts=(connect_attempts + 1)
-              increment_retry_period!
-              ::NewRelic::Agent.logger.warn "Will re-attempt in #{connect_retry_period} seconds"
-              true
-            else
-              disconnect
-              false
-            end
+          def note_connect_failure
+            self.connect_attempts += 1
           end
 
           # When we have a problem connecting to the server, we need
@@ -744,24 +729,12 @@ module NewRelic
             Agent.config.apply_config(server_config, 1)
             log_connection!(config_data) if @service
 
-            @cross_process_id = Agent.config[:cross_process_id]
-            @cross_process_encoding_key = Agent.config[:encoding_key]
-            @cross_process_encoding_bytes = get_bytes(@cross_process_encoding_key) unless @cross_process_encoding_key.nil?
+            # If you're adding something else here to respond to the server-side config,
+            # use Agent.instance.events.subscribe(:finished_configuring) callback instead!
 
             @beacon_configuration = BeaconConfiguration.new
           end
 
-          # Ruby 1.8.6 doesn't support the bytes method on strings.
-          def get_bytes(value)
-            return [] if value.nil?
-
-            bytes = []
-            value.each_byte do |b|
-              bytes << b
-            end
-            bytes
-          end
-
           # Logs when we connect to the server, for debugging purposes
           # - makes sure we know if an agent has not connected
           def log_connection!(config_data)
@@ -825,7 +798,7 @@ module NewRelic
         public :merge_data_from
 
         # Connect to the server and validate the license.  If successful,
-        # @connected has true when finished.  If not successful, you can
+        # connected? returns true when finished.  If not successful, you can
         # keep calling this.  Return false if we could not establish a
         # connection with the server and we should not retry, such as if
         # there's a bad license key.
@@ -840,25 +813,27 @@ module NewRelic
         # * <tt>force_reconnect => true</tt> if you want to establish a new connection
         #   to the server before running the worker loop.  This means you get a separate
         #   agent run and New Relic sees it as a separate instance (default is false).
-        def connect(options)
-          # Don't proceed if we already connected (@connected=true) or if we tried
-          # to connect and were rejected with prejudice because of a license issue
-          # (@connected=false), unless we're forced to by force_reconnect.
-          return if tried_to_connect?(options)
+        def connect(options={})
+          defaults = {
+            :keep_retrying => true,
+            :force_reconnect => false
+          }
+          opts = defaults.merge(options)
 
-          # wait a few seconds for the web server to boot, necessary in development
-          @connect_retry_period = should_keep_retrying?(options) ? 10 : 0
+          return unless should_connect?(opts[:force_reconnect])
 
-          sleep connect_retry_period
           ::NewRelic::Agent.logger.debug "Connecting Process to New Relic: #$0"
           query_server_for_configuration
           @connected_pid = $$
-          @connected = true
+          @connect_state = :connected
         rescue NewRelic::Agent::LicenseException => e
           handle_license_error(e)
         rescue Timeout::Error, StandardError => e
           log_error(e)
-          if should_retry?
+          if opts[:keep_retrying]
+            note_connect_failure
+            ::NewRelic::Agent.logger.warn "Will re-attempt in #{connect_retry_period} seconds"
+            sleep connect_retry_period
             retry
           else
             disconnect
@@ -876,13 +851,6 @@ module NewRelic
           control.root
         end
 
-        # Checks whether this process is a Passenger or Unicorn
-        # spawning server - if so, we probably don't intend to report
-        # statistics from this process
-        def is_application_spawner?
-          $0 =~ /ApplicationSpawner|^unicorn\S* master/
-        end
-
         # calls the busy harvester and collects timeslice data to
         # send later
         def harvest_timeslice_data(time=Time.now)
@@ -1071,7 +1039,7 @@ module NewRelic
         # If this process comes from a parent process, it will not
         # disconnect, so that the parent process can continue to send data
         def graceful_disconnect
-          if @connected
+          if connected?
             begin
               @service.request_timeout = 10
               transmit_data(true)

data/lib/new_relic/agent/configuration/manager.rb

@@ -20,10 +20,14 @@ module NewRelic
         end
 
         def apply_config(source, level=0)
+          was_finished = finished_configuring?
+
           invoke_callbacks(:add, source)
           @config_stack.insert(level, source.freeze)
           reset_cache
           log_config(:add, source)
+
+          notify_finished_configuring if !was_finished && finished_configuring?
         end
 
         def remove_config(source=nil)
@@ -86,6 +90,14 @@ module NewRelic
           end
         end
 
+        def notify_finished_configuring
+          NewRelic::Agent.instance.events.notify(:finished_configuring)
+        end
+
+        def finished_configuring?
+          @config_stack.any? {|s| s.is_a?(ServerSource)}
+        end
+
         def flattened
           @config_stack.reverse.inject({}) do |flat,layer|
             thawed_layer = layer.dup

data/lib/new_relic/agent/configuration/yaml_source.rb

@@ -25,7 +25,11 @@ module NewRelic
             license_key = ''
 
             erb = ERB.new(file).result(binding)
-            config = merge!(YAML.load(erb)[env] || {})
+            confighash = YAML.load(erb)
+            ::NewRelic::Agent.logger.error("Config (#{path}) doesn't include a '#{env}' environment!") unless
+              confighash.key?(env)
+
+            config = merge!(confighash[env] || {})
           rescue ScriptError, StandardError => e
             ::NewRelic::Agent.logger.error("Unable to read configuration file #{path}: #{e}")
           end

data/lib/new_relic/agent/cross_process_monitoring.rb

@@ -1,43 +1,187 @@
+require 'new_relic/rack/agent_hooks'
+require 'new_relic/agent/thread'
+
 module NewRelic
   module Agent
-    module CrossProcessMonitoring
+    class CrossProcessMonitor
+
+      def initialize(events = nil)
+        # When we're starting up for real in the agent, we get passed the events
+        # Other spots can pull from the agent, during startup the agent doesn't exist yet!
+        events ||= Agent.instance.events
+        @trusted_ids = []
+
+        events.subscribe(:finished_configuring) do
+          finish_setup(Agent.config)
+          register_event_listeners
+        end
+      end
+
+      def finish_setup(config)
+        @cross_process_id = config[:cross_process_id]
+        @encoding_key = config[:encoding_key]
+        @encoding_bytes = get_bytes(@encoding_key) unless @encoding_key.nil?
+        @trusted_ids = config[:trusted_account_ids] || []
+      end
+
+      # Expected sequence of events:
+      #   :before_call will save our cross process request id to the thread
+      #   :start_transaction will get called when a transaction starts up
+      #   :after_call will write our response headers/metrics and clean up the thread
+      def register_event_listeners
+        NewRelic::Agent.logger.debug("Wiring up Cross Process monitoring to events after finished configuring")
+
+        events = Agent.instance.events
+        events.subscribe(:before_call) do |env|
+          save_client_cross_process_id(env)
+        end
+
+        events.subscribe(:start_transaction) do |name|
+          set_transaction_custom_parameters
+        end
+
+        events.subscribe(:after_call) do |env, (status_code, headers, body)|
+          insert_response_header(env, headers)
+        end
+
+        events.subscribe(:notice_error) do |_, options|
+          set_error_custom_parameters(options)
+        end
+      end
+
+      # Because we aren't in the right spot when our transaction actually
+      # starts, hold client_cross_process_id we get thread local until then.
+      THREAD_ID_KEY = :newrelic_client_cross_process_id
+
+      def save_client_cross_process_id(request_headers)
+        if should_process_request(request_headers)
+          NewRelic::Agent::AgentThread.current[THREAD_ID_KEY] = decoded_id(request_headers)
+        end
+      end
 
-      module_function
+      def clear_client_cross_process_id
+        NewRelic::Agent::AgentThread.current[THREAD_ID_KEY] = nil
+      end
 
-      def insert_response_header(request, response)
-        if Agent.config[:'cross_process.enabled'] &&
-            NewRelic::Agent.instance.cross_process_id && (id = id_from_request(request))
+      def client_cross_process_id
+        NewRelic::Agent::AgentThread.current[THREAD_ID_KEY]
+      end
 
-          content_length = -1
+      def insert_response_header(request_headers, response_headers)
+        unless client_cross_process_id.nil?
           timings = NewRelic::Agent::BrowserMonitoring.timings
+          content_length = content_length_from_request(request_headers)
+
+          set_response_headers(response_headers, timings, content_length)
+          set_metrics(client_cross_process_id, timings)
+
+          clear_client_cross_process_id
+        end
+      end
+
+      def should_process_request(request_headers)
+        return Agent.config[:'cross_process.enabled'] &&
+            @cross_process_id &&
+            trusts?(request_headers)
+      end
+
+      # Expects an ID of format "12#345", and will only accept that!
+      def trusts?(request)
+        id = decoded_id(request)
+        split_id = id.match(/(\d+)#\d+/)
+        return false if split_id.nil?
 
-          # FIXME the transaction name might not be properly encoded.  use a json generator
-          payload = %[["#{NewRelic::Agent.instance.cross_process_id}","#{timings.transaction_name}",#{timings.queue_time_in_millis},#{timings.app_time_in_millis},#{content_length}] ]
-          payload = obfuscate_with_key(payload, NewRelic::Agent.instance.cross_process_encoding_bytes)
+        @trusted_ids.include?(split_id.captures.first.to_i)
+      end
+
+      def set_response_headers(response_headers, timings, content_length)
+        response_headers['X-NewRelic-App-Data'] = build_payload(timings, content_length)
+      end
+
+      def build_payload(timings, content_length)
+
+        # FIXME The transaction name might not be properly encoded.  use a json generator
+        # For now we just handle quote characters by dropping them
+        transaction_name = timings.transaction_name.gsub(/["']/, "")
+
+        payload = %[["#{@cross_process_id}","#{transaction_name}",#{timings.queue_time_in_seconds},#{timings.app_time_in_seconds},#{content_length}] ]
+        payload = obfuscate_with_key(payload)
+      end
+
+      def set_transaction_custom_parameters
+        # We expect to get the before call to set the id (if we have it) before
+        # this, and then write our custom parameter when the transaction starts
+        NewRelic::Agent.add_custom_parameters(:client_cross_process_id => client_cross_process_id) unless client_cross_process_id.nil?
+      end
+
+      def set_error_custom_parameters(options)
+        options[:client_cross_process_id] = client_cross_process_id unless client_cross_process_id.nil?
+      end
+
+      def set_metrics(id, timings)
+        metric = NewRelic::Agent.instance.stats_engine.get_stats_no_scope("ClientApplication/#{id}/all")
+        metric.record_data_point(timings.app_time_in_seconds)
+      end
+
+      def obfuscate_with_key(text)
+        Base64.encode64(encode_with_key(text)).chomp
+      end
 
-          response['X-NewRelic-App-Data'] = payload
-          #FIXME generate ClientApplication metric.  id must be decoded first
-          # String metricName = MessageFormat.format("ClientApplication/{0}/all", id);
+      def decode_with_key(text)
+        encode_with_key(Base64.decode64(text))
+      end
+
+      NEWRELIC_ID_HEADER_KEYS = %w{X-NewRelic-ID HTTP_X_NEWRELIC_ID X_NEWRELIC_ID}
+      CONTENT_LENGTH_HEADER_KEYS = %w{Content-Length HTTP_CONTENT_LENGTH CONTENT_LENGTH}
+
+      def decoded_id(request)
+        encoded_id = from_headers(request, NEWRELIC_ID_HEADER_KEYS)
+        return "" if encoded_id.nil?
+
+        decode_with_key(encoded_id)
+      end
+
+      def content_length_from_request(request)
+        from_headers(request, CONTENT_LENGTH_HEADER_KEYS) || -1
+      end
+
+
+
+      private
+
+      # Ruby 1.8.6 doesn't support the bytes method on strings.
+      def get_bytes(value)
+        return [] if value.nil?
+
+        bytes = []
+        value.each_byte do |b|
+          bytes << b
         end
+        bytes
       end
 
-      def obfuscate_with_key(text, key_bytes)
-        obfuscated = ""
+      def encode_with_key(text)
+        key_bytes =  @encoding_bytes
+
+        encoded = ""
         index = 0
         text.each_byte{|byte|
-          obfuscated.concat((byte ^ key_bytes[index % key_bytes.length].to_i))
+          encoded.concat((byte ^ key_bytes[index % key_bytes.length].to_i))
           index+=1
         }
-
-        [obfuscated].pack("m0").gsub("\n", '')
+        encoded
       end
 
-      def id_from_request(request)
-        %w{X-NewRelic-ID HTTP_X_NEWRELIC_ID X_NEWRELIC_ID}.each do |header|
-          return request.env[header] if request.env.has_key?(header)
+      def from_headers(request, try_keys)
+        # For lookups, upcase all our keys on both sides just to be safe
+        upcased_keys = try_keys.map{|k| k.upcase}
+        upcased_keys.each do |header|
+          found_key = request.keys.find { |k| k.upcase == header }
+          return request[found_key] unless found_key.nil?
         end
         nil
       end
+
     end
   end
 end