newrelic_rpm 6.6.0.358 → 6.11.0

data/lib/new_relic/agent/agent.rb

@@ -8,6 +8,8 @@ require 'net/http'
 require 'logger'
 require 'zlib'
 require 'stringio'
+require 'new_relic/constants'
+require 'new_relic/coerce'
 require 'new_relic/agent/autostart'
 require 'new_relic/agent/harvester'
 require 'new_relic/agent/hostname'
@@ -17,9 +19,8 @@ require 'new_relic/agent/configuration/manager'
 require 'new_relic/agent/database'
 require 'new_relic/agent/commands/agent_command_router'
 require 'new_relic/agent/event_listener'
-require 'new_relic/agent/cross_app_monitor'
-require 'new_relic/agent/distributed_trace_monitor'
-require 'new_relic/agent/synthetics_monitor'
+require 'new_relic/agent/distributed_tracing'
+require 'new_relic/agent/monitors'
 require 'new_relic/agent/transaction_event_recorder'
 require 'new_relic/agent/custom_event_aggregator'
 require 'new_relic/agent/span_event_aggregator'
@@ -48,31 +49,30 @@ module NewRelic
       def initialize
         @started = false
         @event_loop = nil
+        @worker_thread = nil
 
         @service = NewRelicService.new
 
-        @events                    = NewRelic::Agent::EventListener.new
-        @stats_engine              = NewRelic::Agent::StatsEngine.new
-        @transaction_sampler       = NewRelic::Agent::TransactionSampler.new
-        @sql_sampler               = NewRelic::Agent::SqlSampler.new
-        @agent_command_router      = NewRelic::Agent::Commands::AgentCommandRouter.new(@events)
-        @cross_app_monitor         = NewRelic::Agent::CrossAppMonitor.new(@events)
-        @distributed_trace_monitor = NewRelic::Agent::DistributedTraceMonitor.new(@events)
-        @synthetics_monitor        = NewRelic::Agent::SyntheticsMonitor.new(@events)
-        @error_collector           = NewRelic::Agent::ErrorCollector.new @events
-        @transaction_rules         = NewRelic::Agent::RulesEngine.new
-        @harvest_samplers          = NewRelic::Agent::SamplerCollection.new(@events)
-        @monotonic_gc_profiler     = NewRelic::Agent::VM::MonotonicGCProfiler.new
-        @javascript_instrumentor   = NewRelic::Agent::JavascriptInstrumentor.new(@events)
-        @adaptive_sampler          = NewRelic::Agent::AdaptiveSampler.new(self.class.config[:sampling_target],
-                                                                          self.class.config[:sampling_target_period_in_seconds])
-
-        @harvester       = NewRelic::Agent::Harvester.new(@events)
+        @events                    = EventListener.new
+        @stats_engine              = StatsEngine.new
+        @transaction_sampler       = TransactionSampler.new
+        @sql_sampler               = SqlSampler.new
+        @agent_command_router      = Commands::AgentCommandRouter.new @events
+        @monitors                  = Monitors.new @events
+        @error_collector           = ErrorCollector.new @events
+        @transaction_rules         = RulesEngine.new
+        @harvest_samplers          = SamplerCollection.new @events
+        @monotonic_gc_profiler     = VM::MonotonicGCProfiler.new
+        @javascript_instrumentor   = JavascriptInstrumentor.new @events
+        @adaptive_sampler          = AdaptiveSampler.new(Agent.config[:sampling_target],
+                                                         Agent.config[:sampling_target_period_in_seconds])
+
+        @harvester       = Harvester.new @events
         @after_fork_lock = Mutex.new
 
-        @transaction_event_recorder = NewRelic::Agent::TransactionEventRecorder.new @events
-        @custom_event_aggregator    = NewRelic::Agent::CustomEventAggregator.new @events
-        @span_event_aggregator      = NewRelic::Agent::SpanEventAggregator.new @events
+        @transaction_event_recorder = TransactionEventRecorder.new @events
+        @custom_event_aggregator    = CustomEventAggregator.new @events
+        @span_event_aggregator      = SpanEventAggregator.new @events
 
         @connect_state      = :pending
         @connect_attempts   = 0
@@ -94,7 +94,7 @@ module NewRelic
       end
 
       def refresh_attribute_filter
-        @attribute_filter = NewRelic::Agent::AttributeFilter.new(NewRelic::Agent.config)
+        @attribute_filter = AttributeFilter.new(Agent.config)
       end
 
       # contains all the class-level methods for NewRelic::Agent::Agent
@@ -127,13 +127,16 @@ module NewRelic
         # cross application tracing ids and encoding
         attr_reader :cross_process_id
         attr_reader :cross_app_encoding_bytes
-        attr_reader :cross_app_monitor
         # service for communicating with collector
         attr_accessor :service
         # Global events dispatcher. This will provides our primary mechanism
         # for agent-wide events, such as finishing configuration, error notification
         # and request before/after from Rack.
         attr_reader :events
+
+        # listens and responds to events that need to process headers
+        # for synthetics and distributed tracing
+        attr_reader :monitors
         # Transaction and metric renaming rules as provided by the
         # collector on connect.  The former are applied during txns,
         # the latter during harvest.
@@ -147,6 +150,7 @@ module NewRelic
         attr_reader :transaction_event_recorder
         attr_reader :attribute_filter
         attr_reader :adaptive_sampler
+        attr_reader :environment_report
 
         def transaction_event_aggregator
           @transaction_event_recorder.transaction_event_aggregator
@@ -205,7 +209,7 @@ module NewRelic
         end
 
         def install_pipe_service(channel_id)
-          @service = NewRelic::Agent::PipeService.new(channel_id)
+          @service = PipeService.new(channel_id)
           if connected?
             @connected_pid = Process.pid
           else
@@ -236,12 +240,18 @@ module NewRelic
         end
 
         def revert_to_default_configuration
-          NewRelic::Agent.config.remove_config_type(:manual)
-          NewRelic::Agent.config.remove_config_type(:server)
+          Agent.config.remove_config_type(:manual)
+          Agent.config.remove_config_type(:server)
         end
 
         def stop_event_loop
           @event_loop.stop if @event_loop
+          # Wait the end of the event loop thread.
+          if @worker_thread
+            unless @worker_thread.join(3)
+              ::NewRelic::Agent.logger.debug "Event loop thread did not stop within 3 seconds"
+            end
+          end
         end
 
         def trap_signals_for_litespeed
@@ -332,7 +342,7 @@ module NewRelic
           end
 
           def log_app_name
-            ::NewRelic::Agent.logger.info "Application: #{Agent.config.app_names.join(", ")}"
+            ::NewRelic::Agent.logger.info "Application: #{Agent.config[:app_name].join(", ")}"
           end
 
           def log_ignore_url_regexes
@@ -345,7 +355,7 @@ module NewRelic
 
           # Logs the configured application names
           def app_name_configured?
-            names = Agent.config.app_names
+            names = Agent.config[:app_name]
             return names.respond_to?(:any?) && names.any?
           end
 
@@ -456,11 +466,11 @@ module NewRelic
           def defer_for_resque?
             NewRelic::Agent.config[:dispatcher] == :resque &&
               NewRelic::LanguageSupport.can_fork? &&
-              !NewRelic::Agent::PipeChannelManager.listener.started?
+              !PipeChannelManager.listener.started?
           end
 
           def in_resque_child_process?
-            defined?(@service) && @service.is_a?(NewRelic::Agent::PipeService)
+            defined?(@service) && @service.is_a?(PipeService)
           end
 
           # Sanity-check the agent configuration and start the agent,
@@ -482,6 +492,7 @@ module NewRelic
 
             unless in_resque_child_process?
               install_exit_handler
+              environment_for_connect
               @harvest_samplers.load_samplers unless Agent.config[:disable_samplers]
             end
 
@@ -555,13 +566,16 @@ module NewRelic
         # This is necessary for cases where we're in a forked child and Ruby
         # might be holding locks for background thread that aren't there anymore.
         def reset_objects_with_locks
-          @stats_engine = NewRelic::Agent::StatsEngine.new
+          @stats_engine = StatsEngine.new
         end
 
         def flush_pipe_data
-          if connected? && @service.is_a?(::NewRelic::Agent::PipeService)
+          if connected? && @service.is_a?(PipeService)
             transmit_data
-            transmit_event_data
+            transmit_analytic_event_data
+            transmit_custom_event_data
+            transmit_error_event_data
+            transmit_span_event_data
           end
         end
 
@@ -577,20 +591,46 @@ module NewRelic
 
           LOG_ONCE_KEYS_RESET_PERIOD = 60.0
 
+          # Certain event types may sometimes need to be on the same interval as metrics,
+          # so we will check config assigned in EventHarvestConfig to determine the interval
+          # on which to report them
+          def interval_for event_type
+            interval = Agent.config[:"event_report_period.#{event_type}"]
+            :"#{interval}_second_harvest"
+          end
+
+          ANALYTIC_EVENT_DATA = "analytic_event_data".freeze
+          CUSTOM_EVENT_DATA = "custom_event_data".freeze
+          ERROR_EVENT_DATA = "error_event_data".freeze
+          SPAN_EVENT_DATA = "span_event_data".freeze
+
           def create_and_run_event_loop
+            data_harvest = :"#{Agent.config[:data_report_period]}_second_harvest"
+            event_harvest = :"#{Agent.config[:event_report_period]}_second_harvest"
+
             @event_loop = create_event_loop
-            @event_loop.on(:report_data) do
+            @event_loop.on(data_harvest) do
               transmit_data
             end
-            @event_loop.on(:report_event_data) do
-              transmit_event_data
+
+            @event_loop.on(interval_for ANALYTIC_EVENT_DATA) do
+              transmit_analytic_event_data
+            end
+            @event_loop.on(interval_for CUSTOM_EVENT_DATA) do
+              transmit_custom_event_data
+            end
+            @event_loop.on(interval_for ERROR_EVENT_DATA) do
+              transmit_error_event_data
+            end
+            @event_loop.on(interval_for SPAN_EVENT_DATA) do
+              transmit_span_event_data
             end
             @event_loop.on(:reset_log_once_keys) do
               ::NewRelic::Agent.logger.clear_already_logged
             end
-            @event_loop.fire_every(Agent.config[:data_report_period],       :report_data)
-            @event_loop.fire_every(Agent.config[:event_report_period],      :report_event_data)
-            @event_loop.fire_every(LOG_ONCE_KEYS_RESET_PERIOD,              :reset_log_once_keys)
+            @event_loop.fire_every(Agent.config[:data_report_period], data_harvest)
+            @event_loop.fire_every(Agent.config[:event_report_period], event_harvest)
+            @event_loop.fire_every(LOG_ONCE_KEYS_RESET_PERIOD, :reset_log_once_keys)
 
             @event_loop.run
           end
@@ -674,7 +714,7 @@ module NewRelic
           end
 
           ::NewRelic::Agent.logger.debug "Creating Ruby Agent worker thread."
-          @worker_thread = NewRelic::Agent::Threading::AgentThread.create('Worker Loop') do
+          @worker_thread = Threading::AgentThread.create('Worker Loop') do
             deferred_work!(connection_options)
           end
         end
@@ -760,23 +800,24 @@ module NewRelic
           # require calls in Rails environments, so this method should only
           # be called synchronously from on the main thread.
           def environment_for_connect
-            Agent.config[:send_environment_info] ? Array(EnvironmentReport.new) : []
+            @environment_report ||= Agent.config[:send_environment_info] ? Array(EnvironmentReport.new) : []
           end
 
-          # Constructs and memoizes an event_harvest_config hash to be used in 
+          # Constructs and memoizes an event_harvest_config hash to be used in
           # the payload sent during connect (and reconnect)
           def event_harvest_config
             @event_harvest_config ||= Configuration::EventHarvestConfig.from_config(Agent.config)
           end
 
           # Builds the payload to send to the connect service,
-          # connects, then configures the agent using the response from 
+          # connects, then configures the agent using the response from
           # the connect service
           def connect_to_server
             request_builder = ::NewRelic::Agent::Connect::RequestBuilder.new \
               @service,
               Agent.config,
-              event_harvest_config
+              event_harvest_config,
+              environment_for_connect
             connect_response = @service.connect request_builder.connect_payload
 
             response_handler = ::NewRelic::Agent::Connect::ResponseHandler.new(self, Agent.config)
@@ -990,7 +1031,7 @@ module NewRelic
         end
 
         def harvest_and_send_timeslice_data
-          NewRelic::Agent::TransactionTimeAggregator.harvest!
+          TransactionTimeAggregator.harvest!
           harvest_and_send_from_container(@stats_engine, :metric_data)
         end
 
@@ -1019,17 +1060,20 @@ module NewRelic
         def harvest_and_send_analytic_event_data
           harvest_and_send_from_container(transaction_event_aggregator, :analytic_event_data)
           harvest_and_send_from_container(synthetics_event_aggregator,  :analytic_event_data)
-          harvest_and_send_from_container(@custom_event_aggregator,     :custom_event_data)
         end
 
-        def harvest_and_send_span_event_data
-          harvest_and_send_from_container(span_event_aggregator, :span_event_data)
+        def harvest_and_send_custom_event_data
+          harvest_and_send_from_container(@custom_event_aggregator, :custom_event_data)
         end
 
         def harvest_and_send_error_event_data
           harvest_and_send_from_container @error_collector.error_event_aggregator, :error_event_data
         end
 
+        def harvest_and_send_span_event_data
+          harvest_and_send_from_container(span_event_aggregator, :span_event_data)
+        end
+
         def check_for_and_handle_agent_commands
           begin
             @agent_command_router.check_for_and_handle_agent_commands
@@ -1050,18 +1094,30 @@ module NewRelic
           NewRelic::Agent.record_metric("Supportability/remote_unavailable/#{endpoint.to_s}", 0.0)
         end
 
-        def transmit_event_data
-          transmit_single_data_type(:harvest_and_send_analytic_event_data, "TransactionEvent")
+        TRANSACTION_EVENT = "TransactionEvent".freeze
+        def transmit_analytic_event_data
+          transmit_single_data_type(:harvest_and_send_analytic_event_data, TRANSACTION_EVENT)
+        end
+
+        CUSTOM_EVENT = "CustomEvent".freeze
+        def transmit_custom_event_data
+          transmit_single_data_type(:harvest_and_send_custom_event_data, CUSTOM_EVENT)
+        end
+
+        ERROR_EVENT = "ErrorEvent".freeze
+        def transmit_error_event_data
+          transmit_single_data_type(:harvest_and_send_error_event_data, ERROR_EVENT)
         end
 
+        SPAN_EVENT = "SpanEvent".freeze
         def transmit_span_event_data
-          transmit_single_data_type(:harvest_and_send_span_event_data, "SpanEvent")
+          transmit_single_data_type(:harvest_and_send_span_event_data, SPAN_EVENT)
         end
 
         def transmit_single_data_type(harvest_method, supportability_name)
           now = Time.now
 
-          msg = "Sending #{harvest_method.to_s.gsub("harvest_and_send_", "")} to New Relic Service"
+          msg = "Sending #{supportability_name} data to New Relic Service"
           ::NewRelic::Agent.logger.debug msg
 
           @service.session do # use http keep-alive
@@ -1108,7 +1164,10 @@ module NewRelic
 
               @events.notify(:before_shutdown)
               transmit_data
-              transmit_event_data
+              transmit_analytic_event_data
+              transmit_custom_event_data
+              transmit_error_event_data
+              transmit_span_event_data
 
               if @connected_pid == $$ && !@service.kind_of?(NewRelic::Agent::NewRelicService)
                 ::NewRelic::Agent.logger.debug "Sending New Relic service agent run shutdown message"

data/lib/new_relic/agent/agent_logger.rb

@@ -173,6 +173,10 @@ module NewRelic
         StartupLogger.instance.dump(self)
       end
 
+      def self.format_fatal_error message
+        "** [NewRelic] FATAL : #{message}\n"
+      end
+
     end
 
     # In an effort to not lose messages during startup, we trap them in memory

data/lib/new_relic/agent/attribute_filter.rb

@@ -121,7 +121,7 @@ module NewRelic
         @high_security = config[:high_security]
 
         setup_key_cache
-        cache_prefix_blacklist
+        cache_prefix_denylist
       end
 
       # Note the key_cache is a global cache, accessible by multiple threads,
@@ -222,16 +222,16 @@ module NewRelic
       # arguments for Sidekiq and Resque in the common case, since none of
       # these attributes are captured by default.
       #
-      def cache_prefix_blacklist
-        @prefix_blacklist = {}
-        @prefix_blacklist[:'request.parameters'] = true unless might_allow_prefix_uncached?(:'request.parameters')
-        @prefix_blacklist[:'job.sidekiq.args']   = true unless might_allow_prefix_uncached?(:'job.sidekiq.args')
-        @prefix_blacklist[:'job.resque.args']    = true unless might_allow_prefix_uncached?(:'job.resque.args')
+      def cache_prefix_denylist
+        @prefix_denylist = {}
+        @prefix_denylist[:'request.parameters'] = true unless might_allow_prefix_uncached?(:'request.parameters')
+        @prefix_denylist[:'job.sidekiq.args']   = true unless might_allow_prefix_uncached?(:'job.sidekiq.args')
+        @prefix_denylist[:'job.resque.args']    = true unless might_allow_prefix_uncached?(:'job.resque.args')
       end
 
       # Note that the given prefix *must* be a Symbol
       def might_allow_prefix?(prefix)
-        !@prefix_blacklist.include?(prefix)
+        !@prefix_denylist.include?(prefix)
       end
 
       def might_allow_prefix_uncached?(prefix)

data/lib/new_relic/agent/attributes.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'new_relic/agent/attribute_processing'
+
+module NewRelic
+  module Agent
+    class Attributes
+      KEY_LIMIT   = 255
+      VALUE_LIMIT = 255
+      COUNT_LIMIT = 64
+
+      def initialize(filter)
+        @filter = filter
+
+        @custom_attributes = {}
+        @agent_attributes = {}
+        @intrinsic_attributes = {}
+
+        @custom_destinations = {}
+        @agent_destinations = {}
+        @already_warned_count_limit = nil
+      end
+
+      def add_agent_attribute(key, value, default_destinations)
+        destinations = @filter.apply(key, default_destinations)
+        return if destinations == AttributeFilter::DST_NONE
+
+        @agent_destinations[key] = destinations
+        add(@agent_attributes, key, value)
+      end
+
+      def add_agent_attribute_with_key_check(key, value, default_destinations)
+        if exceeds_bytesize_limit? key, KEY_LIMIT
+          NewRelic::Agent.logger.debug("Agent attribute #{key} was dropped for exceeding key length limit #{KEY_LIMIT}")
+          return
+        end
+
+        add_agent_attribute(key, value, default_destinations)
+      end
+
+      def add_intrinsic_attribute(key, value)
+        add(@intrinsic_attributes, key, value)
+      end
+
+      def merge_untrusted_agent_attributes(attributes, prefix, default_destinations)
+        return if @filter.high_security?
+        return if !@filter.might_allow_prefix?(prefix)
+
+        AttributeProcessing.flatten_and_coerce(attributes, prefix) do |k, v|
+          add_agent_attribute_with_key_check(k, v, AttributeFilter::DST_NONE)
+        end
+      end
+
+      def merge_custom_attributes(other)
+        return unless Agent.config[:'custom_attributes.enabled']
+        return if other.empty?
+        AttributeProcessing.flatten_and_coerce(other) do |k, v|
+          add_custom_attribute(k, v)
+        end
+      end
+
+      def custom_attributes_for(destination)
+        for_destination(@custom_attributes, @custom_destinations, destination)
+      end
+
+      def agent_attributes_for(destination)
+        for_destination(@agent_attributes, @agent_destinations, destination)
+      end
+
+      def intrinsic_attributes_for(destination)
+        if destination == NewRelic::Agent::AttributeFilter::DST_TRANSACTION_TRACER ||
+           destination == NewRelic::Agent::AttributeFilter::DST_ERROR_COLLECTOR
+          @intrinsic_attributes
+        else
+          NewRelic::EMPTY_HASH
+        end
+      end
+
+      private
+
+      def add_custom_attribute(key, value)
+        if @custom_attributes.size >= COUNT_LIMIT
+          unless @already_warned_count_limit
+            NewRelic::Agent.logger.warn("Custom attributes count exceeded limit of #{COUNT_LIMIT}. Any additional custom attributes during this transaction will be dropped.")
+            @already_warned_count_limit = true
+          end
+          return
+        end
+
+        if @filter.high_security?
+          NewRelic::Agent.logger.debug("Unable to add custom attribute #{key} while in high security mode.")
+          return
+        end
+
+        if exceeds_bytesize_limit?(key, KEY_LIMIT)
+          NewRelic::Agent.logger.warn("Custom attribute key '#{key}' was longer than limit of #{KEY_LIMIT} bytes. This attribute will be dropped.")
+          return
+        end
+
+        destinations = @filter.apply(key, AttributeFilter::DST_ALL)
+        return if destinations == AttributeFilter::DST_NONE
+
+        @custom_destinations[key] = destinations
+        add(@custom_attributes, key, value)
+      end
+
+      def add(attributes, key, value)
+        return if value.nil?
+
+        if exceeds_bytesize_limit?(value, VALUE_LIMIT)
+          value = slice(value)
+        end
+
+        attributes[key] = value
+      end
+
+      def for_destination(attributes, calculated_destinations, destination)
+        # Avoid allocating anything if there are no attrs at all
+        return NewRelic::EMPTY_HASH if attributes.empty?
+
+        attributes.inject({}) do |memo, (key, value)|
+          if @filter.allows?(calculated_destinations[key], destination)
+            memo[key] = value
+          end
+          memo
+        end
+      end
+
+      def exceeds_bytesize_limit?(value, limit)
+        if value.respond_to?(:bytesize)
+          value.bytesize > limit
+        elsif value.is_a?(Symbol)
+          value.to_s.bytesize > limit
+        else
+          false
+        end
+      end
+
+      # Take one byte past our limit. Why? This lets us unconditionally chop!
+      # the end. It'll either remove the one-character-too-many we have, or
+      # peel off the partial, mangled character left by the byteslice.
+      def slice(incoming)
+        result = incoming.to_s.byteslice(0, VALUE_LIMIT + 1)
+        result.chop!
+      end
+    end
+  end
+end