newrelic_rpm 8.13.0 → 8.14.0

data/lib/new_relic/agent/agent_helpers/connect.rb

@@ -0,0 +1,227 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module AgentHelpers
+      # This module is an artifact of a refactoring of the connect
+      # method - all of its methods are used in that context, so it
+      # can be refactored at will. It should be fully tested
+      module Connect
+        # number of attempts we've made to contact the server
+        attr_accessor :connect_attempts
+
+        # Disconnect just sets the connect state to disconnected, preventing
+        # further retries.
+        def disconnect
+          @connect_state = :disconnected
+          true
+        end
+
+        def connected?
+          @connect_state == :connected
+        end
+
+        def disconnected?
+          @connect_state == :disconnected
+        end
+
+        # Don't connect if we're already connected, or if we tried to connect
+        # and were rejected with prejudice because of a license issue, unless
+        # we're forced to by force_reconnect.
+        def should_connect?(force = false)
+          force || (!connected? && !disconnected?)
+        end
+
+        # Per the spec at
+        # /agents/agent-specs/Collector-Response-Handling.md, retry
+        # connections after a specific backoff sequence to prevent
+        # hammering the server.
+        def connect_retry_period
+          NewRelic::CONNECT_RETRY_PERIODS[connect_attempts] || NewRelic::MAX_RETRY_PERIOD
+        end
+
+        def note_connect_failure
+          self.connect_attempts += 1
+        end
+
+        # When we have a problem connecting to the server, we need
+        # to tell the user what happened, since this is not an error
+        # we can handle gracefully.
+        def log_error(error)
+          ::NewRelic::Agent.logger.error("Error establishing connection with New Relic Service at #{control.server}:",
+            error)
+        end
+
+        # When the server sends us an error with the license key, we
+        # want to tell the user that something went wrong, and let
+        # them know where to go to get a valid license key
+        #
+        # After this runs, it disconnects the agent so that it will
+        # no longer try to connect to the server, saving the
+        # application and the server load
+        def handle_license_error(error)
+          ::NewRelic::Agent.logger.error( \
+            error.message, \
+            "Visit NewRelic.com to obtain a valid license key, or to upgrade your account."
+          )
+          disconnect
+        end
+
+        def handle_unrecoverable_agent_error(error)
+          ::NewRelic::Agent.logger.error(error.message)
+          disconnect
+          shutdown
+        end
+
+        # Checks whether we should send environment info, and if so,
+        # returns the snapshot from the local environment.
+        # Generating the EnvironmentReport has the potential to trigger
+        # require calls in Rails environments, so this method should only
+        # be called synchronously from on the main thread.
+        def environment_for_connect
+          @environment_report ||= Agent.config[:send_environment_info] ? Array(EnvironmentReport.new) : []
+        end
+
+        # Constructs and memoizes an event_harvest_config hash to be used in
+        # the payload sent during connect (and reconnect)
+        def event_harvest_config
+          @event_harvest_config ||= Configuration::EventHarvestConfig.from_config(Agent.config)
+        end
+
+        # Builds the payload to send to the connect service,
+        # connects, then configures the agent using the response from
+        # the connect service
+        def connect_to_server
+          request_builder = ::NewRelic::Agent::Connect::RequestBuilder.new( \
+            @service,
+            Agent.config,
+            event_harvest_config,
+            environment_for_connect
+          )
+          connect_response = @service.connect(request_builder.connect_payload)
+
+          response_handler = ::NewRelic::Agent::Connect::ResponseHandler.new(self, Agent.config)
+          response_handler.configure_agent(connect_response)
+
+          log_connection(connect_response) if connect_response
+          connect_response
+        end
+
+        # Logs when we connect to the server, for debugging purposes
+        # - makes sure we know if an agent has not connected
+        def log_connection(config_data)
+          ::NewRelic::Agent.logger.debug("Connected to NewRelic Service at #{@service.collector.name}")
+          ::NewRelic::Agent.logger.debug("Agent Run       = #{@service.agent_id}.")
+          ::NewRelic::Agent.logger.debug("Connection data = #{config_data.inspect}")
+          if config_data['messages'] && config_data['messages'].any?
+            log_collector_messages(config_data['messages'])
+          end
+        end
+
+        def log_collector_messages(messages)
+          messages.each do |message|
+            ::NewRelic::Agent.logger.send(message['level'].downcase, message['message'])
+          end
+        end
+
+        # apdex_f is always 4 times the apdex_t
+        def apdex_f
+          (4 * Agent.config[:apdex_t]).to_f
+        end
+
+        class WaitOnConnectTimeout < StandardError
+        end
+
+        # Used for testing to let us know we've actually started to wait
+        def waited_on_connect?
+          @waited_on_connect
+        end
+
+        def signal_connected
+          @wait_on_connect_mutex.synchronize do
+            @wait_on_connect_condition.signal
+          end
+        end
+
+        def wait_on_connect(timeout)
+          return if connected?
+
+          @waited_on_connect = true
+          NewRelic::Agent.logger.debug("Waiting on connect to complete.")
+
+          @wait_on_connect_mutex.synchronize do
+            @wait_on_connect_condition.wait(@wait_on_connect_mutex, timeout)
+          end
+
+          unless connected?
+            raise WaitOnConnectTimeout, "Agent was unable to connect in #{timeout} seconds."
+          end
+        end
+
+        def connect_options(options)
+          {
+            keep_retrying: Agent.config[:keep_retrying],
+            force_reconnect: Agent.config[:force_reconnect]
+          }.merge(options)
+        end
+
+        # Establish a connection to New Relic servers.
+        #
+        # By default, if a connection has already been established, this method
+        # will be a no-op.
+        #
+        # @param [Hash] options
+        # @option options [Boolean] :keep_retrying (true)
+        #   If true, this method will block until a connection is successfully
+        #   established, continuing to retry upon failure. If false, this method
+        #   will return after either successfully connecting, or after failing
+        #   once.
+        #
+        # @option options [Boolean] :force_reconnect (false)
+        #   If true, this method will force establishment of a new connection
+        #   with New Relic, even if there is already an existing connection.
+        #   This is useful primarily when re-establishing a new connection after
+        #   forking off from a parent process.
+        #
+        def connect(options = {})
+          opts = connect_options(options)
+          return unless should_connect?(opts[:force_reconnect])
+
+          ::NewRelic::Agent.logger.debug("Connecting Process to New Relic: #$0")
+          connect_to_server
+          @connected_pid = $$
+          @connect_state = :connected
+          signal_connected
+        rescue NewRelic::Agent::ForceDisconnectException => e
+          handle_force_disconnect(e)
+        rescue NewRelic::Agent::LicenseException => e
+          handle_license_error(e)
+        rescue NewRelic::Agent::UnrecoverableAgentException => e
+          handle_unrecoverable_agent_error(e)
+        rescue StandardError, Timeout::Error, NewRelic::Agent::ServerConnectionException => e
+          retry if retry_from_error?(e, opts)
+        rescue Exception => e
+          ::NewRelic::Agent.logger.error("Exception of unexpected type during Agent#connect():", e)
+
+          raise
+        end
+
+        def retry_from_error?(e, opts)
+          # Allow a killed (aborting) thread to continue exiting during shutdown.
+          # See: https://github.com/newrelic/newrelic-ruby-agent/issues/340
+          raise if Thread.current.status == 'aborting'
+
+          log_error(e)
+          return false unless opts[:keep_retrying]
+
+          note_connect_failure
+          ::NewRelic::Agent.logger.info("Will re-attempt in #{connect_retry_period} seconds")
+          sleep(connect_retry_period)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/agent_helpers/harvest.rb

@@ -0,0 +1,153 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module AgentHelpers
+      module Harvest
+        # Harvests data from the given container, sends it to the named endpoint
+        # on the service, and automatically merges back in upon a recoverable
+        # failure.
+        #
+        # The given container should respond to:
+        #
+        #  #harvest!
+        #    returns a payload that contains enumerable collection of data items and
+        #    optional metadata to be sent to the collector.
+        #
+        #  #reset!
+        #    drop any stored data and reset to a clean state.
+        #
+        #  #merge!(payload)
+        #    merge the given payload back into the internal buffer of the
+        #    container, so that it may be harvested again later.
+        #
+        def harvest_and_send_from_container(container, endpoint)
+          payload = harvest_from_container(container, endpoint)
+          sample_count = harvest_size(container, payload)
+          if sample_count > 0
+            NewRelic::Agent.logger.debug("Sending #{sample_count} items to #{endpoint}")
+            send_data_to_endpoint(endpoint, payload, container)
+          end
+        end
+
+        def harvest_size(container, items)
+          if container.respond_to?(:has_metadata?) && container.has_metadata? && !items.empty?
+            items.last.size
+          else
+            items.size
+          end
+        end
+
+        def harvest_from_container(container, endpoint)
+          items = []
+          begin
+            items = container.harvest!
+          rescue => e
+            NewRelic::Agent.logger.error("Failed to harvest #{endpoint} data, resetting. Error: ", e)
+            container.reset!
+          end
+          items
+        end
+
+        def harvest_and_send_timeslice_data
+          TransactionTimeAggregator.harvest!
+          harvest_and_send_from_container(@stats_engine, :metric_data)
+        end
+
+        def harvest_and_send_slowest_sql
+          harvest_and_send_from_container(@sql_sampler, :sql_trace_data)
+        end
+
+        # This handles getting the transaction traces and then sending
+        # them across the wire.  This includes gathering SQL
+        # explanations, stripping out stack traces, and normalizing
+        # SQL.  note that we explain only the sql statements whose
+        # nodes' execution times exceed our threshold (to avoid
+        # unnecessary overhead of running explains on fast queries.)
+        def harvest_and_send_transaction_traces
+          harvest_and_send_from_container(@transaction_sampler, :transaction_sample_data)
+        end
+
+        def harvest_and_send_for_agent_commands
+          harvest_and_send_from_container(@agent_command_router, :profile_data)
+        end
+
+        def harvest_and_send_errors
+          harvest_and_send_from_container(@error_collector.error_trace_aggregator, :error_data)
+        end
+
+        def harvest_and_send_analytic_event_data
+          harvest_and_send_from_container(transaction_event_aggregator, :analytic_event_data)
+          harvest_and_send_from_container(synthetics_event_aggregator, :analytic_event_data)
+        end
+
+        def harvest_and_send_custom_event_data
+          harvest_and_send_from_container(@custom_event_aggregator, :custom_event_data)
+        end
+
+        def harvest_and_send_error_event_data
+          harvest_and_send_from_container(@error_collector.error_event_aggregator, :error_event_data)
+        end
+
+        def harvest_and_send_span_event_data
+          harvest_and_send_from_container(span_event_aggregator, :span_event_data)
+        end
+
+        def harvest_and_send_log_event_data
+          harvest_and_send_from_container(@log_event_aggregator, :log_event_data)
+        end
+
+        def harvest_and_send_data_types
+          harvest_and_send_errors
+          harvest_and_send_error_event_data
+          harvest_and_send_transaction_traces
+          harvest_and_send_slowest_sql
+          harvest_and_send_timeslice_data
+          harvest_and_send_span_event_data
+          harvest_and_send_log_event_data
+        end
+
+        def send_data_to_endpoint(endpoint, payload, container)
+          begin
+            @service.send(endpoint, payload)
+          rescue ForceRestartException, ForceDisconnectException
+            raise
+          rescue SerializationError => e
+            NewRelic::Agent.logger.warn("Failed to serialize data for #{endpoint}, discarding. Error: ", e)
+          rescue UnrecoverableServerException => e
+            NewRelic::Agent.logger.warn("#{endpoint} data was rejected by remote service, discarding. Error: ", e)
+          rescue ServerConnectionException => e
+            log_remote_unavailable(endpoint, e)
+            container.merge!(payload)
+          rescue => e
+            NewRelic::Agent.logger.info("Unable to send #{endpoint} data, will try again later. Error: ", e)
+            container.merge!(payload)
+          end
+        end
+
+        def check_for_and_handle_agent_commands
+          begin
+            @agent_command_router.check_for_and_handle_agent_commands
+          rescue ForceRestartException, ForceDisconnectException
+            raise
+          rescue UnrecoverableServerException => e
+            NewRelic::Agent.logger.warn("get_agent_commands message was rejected by remote service, discarding. " \
+              "Error: ", e)
+          rescue ServerConnectionException => e
+            log_remote_unavailable(:get_agent_commands, e)
+          rescue => e
+            NewRelic::Agent.logger.info("Error during check_for_and_handle_agent_commands, will retry later: ", e)
+          end
+        end
+
+        def log_remote_unavailable(endpoint, e)
+          NewRelic::Agent.logger.debug("Unable to send #{endpoint} data, will try again later. Error: ", e)
+          NewRelic::Agent.record_metric("Supportability/remote_unavailable", 0.0)
+          NewRelic::Agent.record_metric("Supportability/remote_unavailable/#{endpoint.to_s}", 0.0)
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/agent_helpers/shutdown.rb

@@ -0,0 +1,72 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module AgentHelpers
+      module Shutdown
+        # Attempt a graceful shutdown of the agent, flushing any remaining
+        # data.
+        def shutdown
+          return unless started?
+
+          ::NewRelic::Agent.logger.info("Starting Agent shutdown")
+
+          stop_event_loop
+          trap_signals_for_litespeed
+          untraced_graceful_disconnect
+          revert_to_default_configuration
+
+          @started = nil
+          Control.reset
+        end
+
+        def untraced_graceful_disconnect
+          begin
+            NewRelic::Agent.disable_all_tracing do
+              graceful_disconnect
+            end
+          rescue => e
+            ::NewRelic::Agent.logger.error(e)
+          end
+        end
+
+        # This method contacts the server to send remaining data and
+        # let the server know that the agent is shutting down - this
+        # allows us to do things like accurately set the end of the
+        # lifetime of the process
+        #
+        # If this process comes from a parent process, it will not
+        # disconnect, so that the parent process can continue to send data
+        def graceful_disconnect
+          if connected?
+            begin
+              @service.request_timeout = 10
+
+              @events.notify(:before_shutdown)
+              transmit_data_types
+              shutdown_service
+
+              ::NewRelic::Agent.logger.debug("Graceful disconnect complete")
+            rescue Timeout::Error, StandardError => e
+              ::NewRelic::Agent.logger.debug("Error when disconnecting #{e.class.name}: #{e.message}")
+            end
+          else
+            ::NewRelic::Agent.logger.debug("Bypassing graceful disconnect - agent not connected")
+          end
+        end
+
+        def shutdown_service
+          if @connected_pid == $$ && !@service.kind_of?(NewRelic::Agent::NewRelicService)
+            ::NewRelic::Agent.logger.debug("Sending New Relic service agent run shutdown message")
+            @service.shutdown
+          else
+            ::NewRelic::Agent.logger.debug("This agent connected from parent process #{@connected_pid}--not sending " \
+              "shutdown")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/agent_helpers/special_startup.rb

@@ -0,0 +1,75 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module AgentHelpers
+      module SpecialStartup
+        # If we're using a dispatcher that forks before serving
+        # requests, we need to wait until the children are forked
+        # before connecting, otherwise the parent process sends useless data
+        def using_forking_dispatcher?
+          # TODO: MAJOR VERSION - remove :rainbows
+          if [:puma, :passenger, :rainbows, :unicorn].include?(Agent.config[:dispatcher])
+            ::NewRelic::Agent.logger.info("Deferring startup of agent reporting thread because " \
+              "#{Agent.config[:dispatcher]} may fork.")
+            true
+          else
+            false
+          end
+        end
+
+        # Return true if we're using resque and it hasn't had a chance to (potentially)
+        # daemonize itself. This avoids hanging when there's a Thread started
+        # before Resque calls Process.daemon (Jira RUBY-857)
+        def defer_for_resque?
+          NewRelic::Agent.config[:dispatcher] == :resque &&
+            NewRelic::Agent::Instrumentation::Resque::Helper.resque_fork_per_job? &&
+            !PipeChannelManager.listener.started?
+        end
+
+        def in_resque_child_process?
+          defined?(@service) && @service.is_a?(PipeService)
+        end
+
+        def defer_for_delayed_job?
+          NewRelic::Agent.config[:dispatcher] == :delayed_job &&
+            !NewRelic::DelayedJobInjection.worker_name
+        end
+
+        # This matters when the following three criteria are met:
+        #
+        # 1. A Sinatra 'classic' application is being run
+        # 2. The app is being run by executing the main file directly, rather
+        #    than via a config.ru file.
+        # 3. newrelic_rpm is required *after* sinatra
+        #
+        # In this case, the entire application runs from an at_exit handler in
+        # Sinatra, and if we were to install ours, it would be executed before
+        # the one in Sinatra, meaning that we'd shutdown the agent too early
+        # and never collect any data.
+        def sinatra_classic_app?
+          (
+            defined?(Sinatra::Application) &&
+            Sinatra::Application.respond_to?(:run) &&
+            Sinatra::Application.run?
+          )
+        end
+
+        def should_install_exit_handler?
+          return false unless Agent.config[:send_data_on_exit]
+
+          !sinatra_classic_app? || Agent.config[:force_install_exit_handler]
+        end
+
+        def install_exit_handler
+          return unless should_install_exit_handler?
+
+          NewRelic::Agent.logger.debug("Installing at_exit handler")
+          at_exit { shutdown }
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/agent_helpers/start_worker_thread.rb

@@ -0,0 +1,167 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module AgentHelpers
+      module StartWorkerThread
+        LOG_ONCE_KEYS_RESET_PERIOD = 60.0
+
+        TRANSACTION_EVENT_DATA = "transaction_event_data".freeze
+        CUSTOM_EVENT_DATA = "custom_event_data".freeze
+        ERROR_EVENT_DATA = "error_event_data".freeze
+        SPAN_EVENT_DATA = "span_event_data".freeze
+        LOG_EVENT_DATA = "log_event_data".freeze
+
+        # Try to launch the worker thread and connect to the server.
+        #
+        # See #connect for a description of connection_options.
+        def start_worker_thread(connection_options = {})
+          if disable = NewRelic::Agent.config[:disable_harvest_thread]
+            NewRelic::Agent.logger.info("Not starting Ruby Agent worker thread because :disable_harvest_thread is " \
+              "#{disable}")
+            return
+          end
+
+          ::NewRelic::Agent.logger.debug("Creating Ruby Agent worker thread.")
+          @worker_thread = Threading::AgentThread.create('Worker Loop') do
+            deferred_work!(connection_options)
+          end
+        end
+
+        def create_event_loop
+          EventLoop.new
+        end
+
+        # If the @worker_thread encounters an error during the attempt to connect to the collector
+        # then the connect attempts enter an exponential backoff retry loop.  To avoid potential
+        # race conditions with shutting down while also attempting to reconnect, we join the
+        # @worker_thread with a timeout threshold.  This allows potentially connecting and flushing
+        # pending data to the server, but without waiting indefinitely for a reconnect to succeed.
+        # The use-case where this typically arises is in cronjob scheduled rake tasks where there's
+        # also some network stability/latency issues happening.
+        def stop_event_loop
+          @event_loop.stop if @event_loop
+          # Wait the end of the event loop thread.
+          if @worker_thread
+            unless @worker_thread.join(3)
+              ::NewRelic::Agent.logger.debug("Event loop thread did not stop within 3 seconds")
+            end
+          end
+        end
+
+        # Certain event types may sometimes need to be on the same interval as metrics,
+        # so we will check config assigned in EventHarvestConfig to determine the interval
+        # on which to report them
+        def interval_for(event_type)
+          interval = Agent.config[:"event_report_period.#{event_type}"]
+          :"#{interval}_second_harvest"
+        end
+
+        def create_and_run_event_loop
+          data_harvest = :"#{Agent.config[:data_report_period]}_second_harvest"
+          event_harvest = :"#{Agent.config[:event_report_period]}_second_harvest"
+
+          @event_loop = create_event_loop
+          @event_loop.on(data_harvest) do
+            transmit_data
+          end
+
+          @event_loop.on(interval_for(TRANSACTION_EVENT_DATA)) do
+            transmit_analytic_event_data
+          end
+          @event_loop.on(interval_for(CUSTOM_EVENT_DATA)) do
+            transmit_custom_event_data
+          end
+          @event_loop.on(interval_for(ERROR_EVENT_DATA)) do
+            transmit_error_event_data
+          end
+          @event_loop.on(interval_for(SPAN_EVENT_DATA)) do
+            transmit_span_event_data
+          end
+          @event_loop.on(interval_for(LOG_EVENT_DATA)) do
+            transmit_log_event_data
+          end
+
+          @event_loop.on(:reset_log_once_keys) do
+            ::NewRelic::Agent.logger.clear_already_logged
+          end
+          @event_loop.fire_every(Agent.config[:data_report_period], data_harvest)
+          @event_loop.fire_every(Agent.config[:event_report_period], event_harvest)
+          @event_loop.fire_every(LOG_ONCE_KEYS_RESET_PERIOD, :reset_log_once_keys)
+
+          @event_loop.run
+        end
+
+        # Handles the case where the server tells us to restart -
+        # this clears the data, clears connection attempts, and
+        # waits a while to reconnect.
+        def handle_force_restart(error)
+          ::NewRelic::Agent.logger.debug(error.message)
+          drop_buffered_data
+          @service.force_restart if @service
+          @connect_state = :pending
+          sleep(30)
+        end
+
+        # when a disconnect is requested, stop the current thread, which
+        # is the worker thread that gathers data and talks to the
+        # server.
+        def handle_force_disconnect(error)
+          ::NewRelic::Agent.logger.warn("Agent received a ForceDisconnectException from the server, disconnecting. " \
+            "(#{error.message})")
+          disconnect
+        end
+
+        # Handles an unknown error in the worker thread by logging
+        # it and disconnecting the agent, since we are now in an
+        # unknown state.
+        def handle_other_error(error)
+          ::NewRelic::Agent.logger.error("Unhandled error in worker thread, disconnecting.")
+          # These errors are fatal (that is, they will prevent the agent from
+          # reporting entirely), so we really want backtraces when they happen
+          ::NewRelic::Agent.logger.log_exception(:error, error)
+          disconnect
+        end
+
+        # a wrapper method to handle all the errors that can happen
+        # in the connection and worker thread system. This
+        # guarantees a no-throw from the background thread.
+        def catch_errors
+          yield
+        rescue NewRelic::Agent::ForceRestartException => e
+          handle_force_restart(e)
+          retry
+        rescue NewRelic::Agent::ForceDisconnectException => e
+          handle_force_disconnect(e)
+        rescue => e
+          handle_other_error(e)
+        end
+
+        # This is the method that is run in a new thread in order to
+        # background the harvesting and sending of data during the
+        # normal operation of the agent.
+        #
+        # Takes connection options that determine how we should
+        # connect to the server, and loops endlessly - typically we
+        # never return from this method unless we're shutting down
+        # the agent
+        def deferred_work!(connection_options)
+          catch_errors do
+            NewRelic::Agent.disable_all_tracing do
+              connect(connection_options)
+              if connected?
+                create_and_run_event_loop
+                # never reaches here unless there is a problem or
+                # the agent is exiting
+              else
+                ::NewRelic::Agent.logger.debug("No connection.  Worker thread ending.")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end