newrelic_rpm 4.0.0.332 → 6.13.1

data/lib/new_relic/agent/method_tracer_helpers.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 
 module NewRelic
   module Agent
@@ -9,80 +9,27 @@ module NewRelic
 
       extend self
 
-      # helper for logging errors to the newrelic_agent.log
-      # properly. Logs the error at error level
-      def log_errors(code_area)
-        yield
-      rescue => e
-        ::NewRelic::Agent.logger.error("Caught exception in #{code_area}.", e)
-      end
-
-      def trace_execution_scoped_header(state, t0)
-        log_errors(:trace_execution_scoped_header) do
-          stack = state.traced_method_stack
-          stack.push_frame(state, :method_tracer, t0)
-        end
-      end
-
-      def record_metrics(state, first_name, other_names, duration, exclusive, options)
-        record_scoped_metric = options.has_key?(:scoped_metric) ? options[:scoped_metric] : true
-        stat_engine = NewRelic::Agent.instance.stats_engine
-        if record_scoped_metric
-          stat_engine.record_scoped_and_unscoped_metrics(state, first_name, other_names, duration, exclusive)
-        else
-          metrics = [first_name].concat(other_names)
-          stat_engine.record_unscoped_metrics(state, metrics, duration, exclusive)
-        end
-      end
-
-      def trace_execution_scoped_footer(state, t0, first_name, metric_names, expected_frame, options, t1=Time.now.to_f)
-        log_errors(:trace_method_execution_footer) do
-          if expected_frame
-            stack = state.traced_method_stack
-            create_metrics = options.has_key?(:metric) ? options[:metric] : true
-            frame = stack.pop_frame(state, expected_frame, first_name, t1, create_metrics)
-            if create_metrics
-              duration = t1 - t0
-              exclusive = duration - frame.children_time
-
-              if duration < MAX_ALLOWED_METRIC_DURATION
-                if duration < 0
-                  ::NewRelic::Agent.logger.log_once(:warn, "metric_duration_negative:#{first_name}",
-                    "Metric #{first_name} has negative duration: #{duration} s")
-                end
-
-                if exclusive < 0
-                  ::NewRelic::Agent.logger.log_once(:warn, "metric_exclusive_negative:#{first_name}",
-                    "Metric #{first_name} has negative exclusive time: duration = #{duration} s, child_time = #{frame.children_time}")
-                end
-
-                record_metrics(state, first_name, metric_names, duration, exclusive, options)
-              else
-                ::NewRelic::Agent.logger.log_once(:warn, "too_huge_metric:#{first_name}",
-                  "Ignoring metric #{first_name} with unacceptably large duration: #{duration} s")
-              end
-            end
-          end
-        end
-      end
-
-      def trace_execution_scoped(metric_names, options={}) #THREAD_LOCAL_ACCESS
-        state = NewRelic::Agent::TransactionState.tl_get
+      def trace_execution_scoped(metric_names, options=NewRelic::EMPTY_HASH) #THREAD_LOCAL_ACCESS
+        state = NewRelic::Agent::Tracer.state
         return yield unless state.is_execution_traced?
 
         metric_names = Array(metric_names)
         first_name   = metric_names.shift
         return yield unless first_name
 
-        segment = NewRelic::Agent::Transaction.start_segment first_name, metric_names
+        segment = NewRelic::Agent::Tracer.start_segment(
+          name: first_name,
+          unscoped_metrics: metric_names
+        )
+
         if options[:metric] == false
           segment.record_metrics = false
         end
 
         begin
-          yield
+          Tracer.capture_segment_error(segment) { yield }
         ensure
-          segment.finish
+          segment.finish if segment
         end
       end
     end

data/lib/new_relic/agent/monitors.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+
+require_relative 'monitors/inbound_request_monitor'
+
+require_relative 'monitors/synthetics_monitor'
+
+require_relative 'monitors/cross_app_monitor'
+require_relative 'monitors/distributed_tracing_monitor'
+
+module NewRelic
+  module Agent
+    class Monitors
+      attr_reader :cross_app_monitor
+      attr_reader :synthetics_monitor
+      attr_reader :distributed_tracing_monitor
+
+      def initialize events
+        @synthetics_monitor = NewRelic::Agent::SyntheticsMonitor.new events
+        @cross_app_monitor = NewRelic::Agent::DistributedTracing::CrossAppMonitor.new events
+        @distributed_tracing_monitor = NewRelic::Agent::DistributedTracing::Monitor.new events
+      end
+
+    end
+  end
+end

data/lib/new_relic/agent/monitors/cross_app_monitor.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+
+require 'digest'
+require 'json'
+
+require 'new_relic/agent/tracer'
+require 'new_relic/agent/threading/agent_thread'
+
+module NewRelic
+  module Agent
+    module DistributedTracing
+      class CrossAppMonitor < InboundRequestMonitor
+
+        NEWRELIC_ID_HEADER      = 'X-NewRelic-ID'.freeze
+        NEWRELIC_TXN_HEADER     = 'X-NewRelic-Transaction'.freeze
+        NEWRELIC_APPDATA_HEADER = 'X-NewRelic-App-Data'.freeze
+
+        NEWRELIC_ID_HEADER_KEY    = 'HTTP_X_NEWRELIC_ID'.freeze
+        NEWRELIC_TXN_HEADER_KEY   = 'HTTP_X_NEWRELIC_TRANSACTION'.freeze
+        CONTENT_LENGTH_HEADER_KEY = 'HTTP_CONTENT_LENGTH'.freeze
+
+        def on_finished_configuring(events)
+          register_event_listeners(events)
+        end
+
+        def path_hash(txn_name, seed)
+          rotated    = ((seed << 1) | (seed >> 31)) & 0xffffffff
+          app_name   = NewRelic::Agent.config[:app_name].first
+          identifier = "#{app_name};#{txn_name}"
+          sprintf("%08x", rotated ^ hash_transaction_name(identifier))
+        end
+
+        private
+
+        # Expected sequence of events:
+        #   :before_call will save our cross application request id to the thread
+        #   :after_call will write our response headers/metrics and clean up the thread
+        def register_event_listeners(events)
+          NewRelic::Agent.logger.
+            debug("Wiring up Cross Application Tracing to events after finished configuring")
+
+          events.subscribe(:before_call) do |env| #THREAD_LOCAL_ACCESS
+            if id = decoded_id(env) and should_process_request?(id)
+              state = NewRelic::Agent::Tracer.state
+
+              if (txn = state.current_transaction)
+                transaction_info = referring_transaction_info(state, env)
+
+                payload = CrossAppPayload.new(id, txn, transaction_info)
+                txn.distributed_tracer.cross_app_payload = payload
+              end
+
+              CrossAppTracing.assign_intrinsic_transaction_attributes state
+            end
+          end
+
+          events.subscribe(:after_call) do |env, (_status_code, headers, _body)| #THREAD_LOCAL_ACCESS
+            state = NewRelic::Agent::Tracer.state
+
+            insert_response_header(state, env, headers)
+          end
+        end
+
+        def referring_transaction_info(state, request_headers)
+          txn_header = request_headers[NEWRELIC_TXN_HEADER_KEY] or return
+          deserialize_header(txn_header, NEWRELIC_TXN_HEADER)
+        end
+
+        def insert_response_header(state, request_headers, response_headers)
+          txn = state.current_transaction
+          unless txn.nil? || txn.distributed_tracer.cross_app_payload.nil?
+            txn.freeze_name_and_execute_if_not_ignored do
+              content_length = content_length_from_request(request_headers)
+              set_response_headers(txn, response_headers, content_length)
+            end
+          end
+        end
+
+        def should_process_request? id
+          CrossAppTracing.cross_app_enabled? && CrossAppTracing.trusts?(id)
+        end
+
+        def set_response_headers(transaction, response_headers, content_length)
+          payload = obfuscator.obfuscate(
+            ::JSON.dump(
+              transaction.distributed_tracer.cross_app_payload.as_json_array(content_length)))
+
+          response_headers[NEWRELIC_APPDATA_HEADER] = payload
+        end
+
+        def decoded_id(request)
+          encoded_id = request[NEWRELIC_ID_HEADER_KEY]
+          return "" if encoded_id.nil? || encoded_id.empty?
+
+          obfuscator.deobfuscate(encoded_id)
+        end
+
+        def content_length_from_request(request)
+          request[CONTENT_LENGTH_HEADER_KEY] || -1
+        end
+
+        def hash_transaction_name(identifier)
+          Digest::MD5.digest(identifier).unpack("@12N").first & 0xffffffff
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/monitors/distributed_tracing_monitor.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+
+module NewRelic
+  module Agent
+    module DistributedTracing
+      class Monitor < InboundRequestMonitor
+        def on_finished_configuring(events)
+          return unless NewRelic::Agent.config[:'distributed_tracing.enabled']
+          events.subscribe(:before_call, &method(:on_before_call))
+        end
+
+        def on_before_call(request)
+          unless NewRelic::Agent.config[:'distributed_tracing.enabled']
+            NewRelic::Agent.logger.warn "Not configured to accept distributed trace headers" 
+            return
+          end
+
+          return unless txn = Tracer.current_transaction
+
+          txn.distributed_tracer.accept_incoming_request request
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/{inbound_request_monitor.rb → monitors/inbound_request_monitor.rb}

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 
 # This class serves as the base for objects wanting to monitor and respond to
 # incoming web requests. Examples include cross application tracing and
@@ -20,15 +20,15 @@ module NewRelic
       attr_reader :obfuscator
 
       def initialize(events)
-        events.subscribe(:finished_configuring) do
-          # This requires :encoding_key, so must wait until :finished_configuring
+        events.subscribe(:initial_configuration_complete) do
+          # This requires :encoding_key, so must wait until :initial_configuration_complete
           setup_obfuscator
           on_finished_configuring(events)
         end
       end
 
       def setup_obfuscator
-        @obfuscator = NewRelic::Agent::Obfuscator.new(NewRelic::Agent.config[:encoding_key])
+        @obfuscator = Obfuscator.new(Agent.config[:encoding_key])
       end
 
       def deserialize_header(encoded_header, key)

data/lib/new_relic/agent/monitors/synthetics_monitor.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+
+module NewRelic
+  module Agent
+    class SyntheticsMonitor < InboundRequestMonitor
+      SYNTHETICS_HEADER_KEY  = 'HTTP_X_NEWRELIC_SYNTHETICS'.freeze
+
+      SUPPORTED_VERSION = 1
+      EXPECTED_PAYLOAD_LENGTH = 5
+
+      def on_finished_configuring(events)
+        events.subscribe(:before_call, &method(:on_before_call))
+      end
+
+      def on_before_call(request) #THREAD_LOCAL_ACCESS
+        encoded_header = request[SYNTHETICS_HEADER_KEY]
+        return unless encoded_header
+
+        incoming_payload = deserialize_header(encoded_header, SYNTHETICS_HEADER_KEY)
+
+        return unless incoming_payload &&
+          SyntheticsMonitor.is_valid_payload?(incoming_payload) &&
+          SyntheticsMonitor.is_supported_version?(incoming_payload) &&
+          SyntheticsMonitor.is_trusted?(incoming_payload)
+
+        txn = Tracer.current_transaction
+        txn.raw_synthetics_header = encoded_header
+        txn.synthetics_payload    = incoming_payload
+      end
+
+      class << self
+
+        def is_supported_version?(incoming_payload)
+          incoming_payload.first == SUPPORTED_VERSION
+        end
+
+        def is_trusted?(incoming_payload)
+          account_id = incoming_payload[1]
+          Agent.config[:trusted_account_ids].include?(account_id)
+        end
+
+        def is_valid_payload?(incoming_payload)
+          incoming_payload.length == EXPECTED_PAYLOAD_LENGTH
+        end
+
+        def reject_messaging_synthetics_header headers
+          headers.reject {|k,_| k == CrossAppTracing::NR_MESSAGE_BROKER_SYNTHETICS_HEADER}
+        end
+
+      end
+
+    end
+  end
+end

data/lib/new_relic/agent/new_relic_service.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 
 require 'zlib'
 require 'timeout'
@@ -8,6 +8,7 @@ require 'new_relic/agent/audit_logger'
 require 'new_relic/agent/new_relic_service/encoders'
 require 'new_relic/agent/new_relic_service/marshaller'
 require 'new_relic/agent/new_relic_service/json_marshaller'
+require 'new_relic/agent/new_relic_service/security_policy_settings'
 
 module NewRelic
   module Agent
@@ -15,7 +16,8 @@ module NewRelic
       # Specifies the version of the agent's communication protocol with
       # the NewRelic hosted site.
 
-      PROTOCOL_VERSION = 14
+      PROTOCOL_VERSION = 17
+
       # 1f147a42: v10 (tag 3.5.3.17)
       # cf0d1ff1: v9 (tag 3.5.0)
       # 14105: v8 (tag 2.10.3)
@@ -30,29 +32,26 @@ module NewRelic
       # underlying TCP connection may be in a bad state.
       CONNECTION_ERRORS = [Timeout::Error, EOFError, SystemCallError, SocketError].freeze
 
-      attr_accessor :request_timeout, :agent_id
-      attr_reader :collector, :marshaller
+      attr_accessor :request_timeout
+      attr_reader :collector, :marshaller, :agent_id
 
       def initialize(license_key=nil, collector=control.server)
         @license_key = license_key
         @collector = collector
+        @configured_collector = collector
         @request_timeout = Agent.config[:timeout]
         @ssl_cert_store = nil
+        @use_bundled_certs = false
         @in_session = nil
         @agent_id = nil
         @shared_tcp_connection = nil
+        @request_headers_map = nil
+        reset_remote_method_uris
 
         @audit_logger = ::NewRelic::Agent::AuditLogger.new
         Agent.config.register_callback(:'audit_log.enabled') do |enabled|
           @audit_logger.enabled = enabled
         end
-        Agent.config.register_callback(:ssl) do |ssl|
-          if !ssl
-            ::NewRelic::Agent.logger.warn("Agent is configured not to use SSL when communicating with New Relic's servers")
-          else
-            ::NewRelic::Agent.logger.debug("Agent is configured to use SSL")
-          end
-        end
 
         Agent.config.register_callback(:marshaller) do |marshaller|
           if marshaller != 'json'
@@ -63,17 +62,48 @@ module NewRelic
         end
       end
 
+      def agent_id=(id)
+        # Remote URIs have the agent run ID in them, so we need to
+        # clear out our cached values whenever the run ID changes.
+        #
+        reset_remote_method_uris
+        @agent_id = id
+      end
+
       def connect(settings={})
-        if host = get_redirect_host
-          @collector = NewRelic::Control.instance.server_from_host(host)
+        @request_headers_map = nil
+        security_policies = nil
+        if response = preconnect
+          if host = response['redirect_host']
+            @collector = NewRelic::Control.instance.server_from_host(host)
+          end
+          if policies = response['security_policies']
+            security_policies = SecurityPolicySettings.preliminary_settings(policies)
+            settings.merge!(security_policies)
+          end
         end
         response = invoke_remote(:connect, [settings])
-        @agent_id = response['agent_run_id']
+        @request_headers_map = response['request_headers_map']
+        self.agent_id = response['agent_run_id']
+        response.merge!(security_policies) if security_policies
         response
       end
 
-      def get_redirect_host
-        invoke_remote(:get_redirect_host)
+      def preconnect
+        token = Agent.config[:security_policies_token]
+
+        if token && !token.empty?
+          response = invoke_remote(:preconnect, [{'security_policies_token' => token, 'high_security' => false}])
+
+          validator = SecurityPolicySettings::Validator.new(response)
+          validator.validate_matching_agent_config!
+
+          response
+        elsif Agent.config[:high_security]
+          invoke_remote(:preconnect, [{'high_security' => true}])
+        else
+          invoke_remote(:preconnect, [{'high_security' => false}])
+        end
       end
 
       def shutdown(time)
@@ -136,10 +166,6 @@ module NewRelic
         invoke_remote(:agent_command_results, [@agent_id, results])
       end
 
-      def get_xray_metadata(xray_ids)
-        invoke_remote(:get_xray_metadata, [@agent_id, *xray_ids])
-      end
-
       def analytic_event_data(data)
         _, items = data
         invoke_remote(:analytic_event_data, [@agent_id, *data],
@@ -159,20 +185,27 @@ module NewRelic
         NewRelic::Agent.record_metric("Supportability/Events/TransactionError/Seen", :count => metadata[:events_seen])
       end
 
+      def span_event_data(data)
+        metadata, items = data
+        invoke_remote(:span_event_data, [@agent_id, *data], :item_count => items.size)
+        NewRelic::Agent.record_metric("Supportability/Events/SpanEvents/Sent", :count => items.size)
+        NewRelic::Agent.record_metric("Supportability/Events/SpanEvents/Seen", :count => metadata[:events_seen])
+      end
+
       # We do not compress if content is smaller than 64kb.  There are
       # problems with bugs in Ruby in some versions that expose us
       # to a risk of segfaults if we compress aggressively.
-      def compress_request_if_needed(data)
+      def compress_request_if_needed(data, endpoint)
         encoding = 'identity'
         if data.size > 64 * 1024
           encoding = Agent.config[:compressed_content_encoding]
-          data = if encoding == 'gzip'
-            Encoders::Compressed::Gzip.encode(data)
-          else
+          data = if encoding == 'deflate'
             Encoders::Compressed::Deflate.encode(data)
+          else
+            Encoders::Compressed::Gzip.encode(data)
           end
         end
-        check_post_size(data)
+        check_post_size(data, endpoint)
         [data, encoding]
       end
 
@@ -259,13 +292,20 @@ module NewRelic
         # installed
         conn.use_ssl     = true
         conn.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        conn.cert_store  = ssl_cert_store
+        set_cert_store(conn)
       rescue StandardError, LoadError
-        msg = "Agent is configured to use SSL, but SSL is not available in the environment. "
-        msg << "Either disable SSL in the agent configuration, or install SSL support."
+        msg = "SSL is not available in the environment; please install SSL support."
         raise UnrecoverableAgentException.new(msg)
       end
 
+      def set_cert_store(conn)
+        if @use_bundled_certs || NewRelic::Agent.config[:ca_bundle_path]
+          conn.cert_store  = ssl_cert_store
+        else
+          ::NewRelic::Agent.logger.debug("Using default security certificates")
+        end
+      end
+
       def start_connection(conn)
         NewRelic::Agent.logger.debug("Opening TCP connection to #{conn.address}:#{conn.port}")
         Timeout.timeout(@request_timeout) { conn.start }
@@ -296,7 +336,7 @@ module NewRelic
           conn = Net::HTTP.new(@collector.name, @collector.port)
         end
 
-        setup_connection_for_ssl(conn) if Agent.config[:ssl]
+        setup_connection_for_ssl(conn)
         setup_connection_timeouts(conn)
 
         ::NewRelic::Agent.logger.debug("Created net/http handle to #{conn.address}:#{conn.port}")
@@ -307,6 +347,14 @@ module NewRelic
         conn = create_http_connection
         start_connection(conn)
         conn
+      rescue Timeout::Error
+        if @use_bundled_certs == false
+          ::NewRelic::Agent.logger.info("Unable to connect. Falling back to bundled security certificates")
+          @use_bundled_certs = true
+          retry
+        else
+          raise
+        end
       end
 
       # The path to the certificate file used to verify the SSL
@@ -316,6 +364,7 @@ module NewRelic
           NewRelic::Agent.logger.warn("Couldn't find CA bundle from configured ca_bundle_path: #{path_override}") unless File.exist? path_override
           path_override
         else
+          ::NewRelic::Agent.increment_metric("Supportability/Ruby/Certificate/BundleRequired")
           File.expand_path(File.join(control.newrelic_root, 'cert', 'cacert.pem'))
         end
       end
@@ -335,11 +384,28 @@ module NewRelic
         NewRelic::Control.instance
       end
 
-      # The path on the server that we should post our data to
-      def remote_method_uri(method, format)
-        params = {'run_id' => @agent_id, 'marshal_format' => format}
-        uri = "/agent_listener/#{PROTOCOL_VERSION}/#{license_key}/#{method}"
-        uri << '?' + params.map do |k,v|
+      def remote_method_uri(method)
+        @remote_method_uris[method]
+      end
+
+      def reset_remote_method_uris
+        @remote_method_uris = Hash.new do |hash, remote_method|
+          hash[remote_method] = generate_remote_method_uri(remote_method)
+        end
+      end
+
+      def generate_remote_method_uri(method)
+        params = {
+          'protocol_version' => PROTOCOL_VERSION,
+          'license_key'      => license_key,
+          'run_id'           => @agent_id,
+          'method'           => method,
+          'marshal_format'   => 'json', # Other formats are explicitly
+                                        # ruled out; see the initializer
+        }
+
+        uri = "/agent_listener/invoke_raw_method?"
+        uri << params.map do |k,v|
           next unless v
           "#{k}=#{v}"
         end.compact.join('&')
@@ -357,7 +423,7 @@ module NewRelic
       def invoke_remote(method, payload = [], options = {})
         start_ts = Time.now
 
-        data, size, serialize_finish_ts = nil
+        data, size, serialize_finish_ts, request_send_ts, response_check_ts = nil
         begin
           data = @marshaller.dump(payload, options)
         rescue StandardError, SystemStackError => e
@@ -365,20 +431,26 @@ module NewRelic
         end
         serialize_finish_ts = Time.now
 
-        data, encoding = compress_request_if_needed(data)
+        data, encoding = compress_request_if_needed(data, method)
         size = data.size
 
-        uri = remote_method_uri(method, @marshaller.format)
-        full_uri = "#{@collector}#{uri}"
+        # Preconnect needs to always use the configured collector host, not the redirect host
+        endpoint_specific_collector = (method == :preconnect) ? @configured_collector : @collector
+
+        uri = remote_method_uri(method)
+        full_uri = "#{endpoint_specific_collector}#{uri}"
 
         @audit_logger.log_request(full_uri, payload, @marshaller)
+        request_send_ts = Time.now
         response = send_request(:data      => data,
                                 :uri       => uri,
                                 :encoding  => encoding,
-                                :collector => @collector)
+                                :collector => endpoint_specific_collector,
+                                :endpoint  => method)
+        response_check_ts = Time.now
         @marshaller.load(decompress_response(response))
       ensure
-        record_timing_supportability_metrics(method, start_ts, serialize_finish_ts)
+        record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
         if size
           record_size_supportability_metrics(method, size, options[:item_count])
         end
@@ -393,11 +465,12 @@ module NewRelic
         raise error
       end
 
-      def record_timing_supportability_metrics(method, start_ts, serialize_finish_ts)
+      def record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
         serialize_time = serialize_finish_ts && (serialize_finish_ts - start_ts)
-        duration = (Time.now - start_ts).to_f
-        NewRelic::Agent.record_metric("Supportability/invoke_remote", duration)
-        NewRelic::Agent.record_metric("Supportability/invoke_remote/#{method.to_s}", duration)
+        request_duration = response_check_ts && (response_check_ts - request_send_ts).to_f
+        if request_duration
+          NewRelic::Agent.record_metric("Supportability/Agent/Collector/#{method.to_s}/Duration", request_duration)
+        end
         if serialize_time
           NewRelic::Agent.record_metric("Supportability/invoke_remote_serialize", serialize_time)
           NewRelic::Agent.record_metric("Supportability/invoke_remote_serialize/#{method.to_s}", serialize_time)
@@ -424,9 +497,10 @@ module NewRelic
 
       # Raises an UnrecoverableServerException if the post_string is longer
       # than the limit configured in the control object
-      def check_post_size(post_string)
-        return if post_string.size < Agent.config[:post_size_limit]
+      def check_post_size(post_string, endpoint)
+        return if post_string.size < Agent.config[:max_payload_size_in_bytes]
         ::NewRelic::Agent.logger.debug "Tried to send too much data: #{post_string.size} bytes"
+        NewRelic::Agent.increment_metric("Supportability/Agent/Collector/#{endpoint}/MaxPayloadSizeLimit")
         raise UnrecoverableServerException.new('413 Request Entity Too Large')
       end
 
@@ -441,10 +515,16 @@ module NewRelic
       #                    contact
       #  - :data => the data to send as the body of the request
       def send_request(opts)
+        headers = {
+          'Content-Encoding' => opts[:encoding],
+          'Host'             => opts[:collector].name
+        }
+        headers.merge!(@request_headers_map) if @request_headers_map
+
         if Agent.config[:put_for_data_send]
-          request = Net::HTTP::Put.new(opts[:uri], 'CONTENT-ENCODING' => opts[:encoding], 'HOST' => opts[:collector].name)
+          request = Net::HTTP::Put.new(opts[:uri], headers)
         else
-          request = Net::HTTP::Post.new(opts[:uri], 'CONTENT-ENCODING' => opts[:encoding], 'HOST' => opts[:collector].name)
+          request = Net::HTTP::Post.new(opts[:uri], headers)
         end
         request['user-agent'] = user_agent
         request.content_type = "application/octet-stream"
@@ -453,6 +533,7 @@ module NewRelic
         response     = nil
         attempts     = 0
         max_attempts = 2
+        endpoint     = opts[:endpoint]
 
         begin
           attempts += 1
@@ -474,20 +555,43 @@ module NewRelic
         log_response(response)
 
         case response
-        when Net::HTTPSuccess
+        when Net::HTTPSuccess,
+             Net::HTTPAccepted
           true # do nothing
-        when Net::HTTPUnauthorized
-          raise LicenseException, 'Invalid license key, please visit support.newrelic.com'
-        when Net::HTTPServiceUnavailable
-          raise ServerConnectionException, "Service unavailable (#{response.code}): #{response.message}"
-        when Net::HTTPGatewayTimeOut
-          raise ServerConnectionException, "Gateway timeout (#{response.code}): #{response.message}"
-        when Net::HTTPRequestEntityTooLarge
-          raise UnrecoverableServerException, '413 Request Entity Too Large'
-        when Net::HTTPUnsupportedMediaType
-          raise UnrecoverableServerException, '415 Unsupported Media Type'
+        when Net::HTTPRequestTimeOut,
+             Net::HTTPTooManyRequests,
+             Net::HTTPInternalServerError,
+             Net::HTTPServiceUnavailable
+          record_endpoint_attempts_supportability_metrics(endpoint)
+          raise ServerConnectionException, "#{response.code}: #{response.message}"
+        when Net::HTTPBadRequest,
+             Net::HTTPForbidden,
+             Net::HTTPNotFound,
+             Net::HTTPMethodNotAllowed,
+             Net::HTTPProxyAuthenticationRequired,
+             Net::HTTPLengthRequired,
+             Net::HTTPRequestEntityTooLarge,
+             Net::HTTPRequestURITooLong,
+             Net::HTTPUnsupportedMediaType,
+             Net::HTTPExpectationFailed,
+             Net::HTTPUnsupportedMediaType,
+             Net::HTTPRequestHeaderFieldsTooLarge
+          record_endpoint_attempts_supportability_metrics(endpoint)
+          record_error_response_supportability_metrics(response.code)
+          raise UnrecoverableServerException, "#{response.code}: #{response.message}"
+        when Net::HTTPConflict,
+             Net::HTTPUnauthorized
+          record_endpoint_attempts_supportability_metrics(endpoint)
+          record_error_response_supportability_metrics(response.code)
+          raise ForceRestartException, "#{response.code}: #{response.message}"
+        when Net::HTTPGone
+          record_endpoint_attempts_supportability_metrics(endpoint)
+          record_error_response_supportability_metrics(response.code)
+          raise ForceDisconnectException, "#{response.code}: #{response.message}"
         else
-          raise ServerConnectionException, "Unexpected response from server (#{response.code}): #{response.message}"
+          record_endpoint_attempts_supportability_metrics(endpoint)
+          record_error_response_supportability_metrics(response.code)
+          raise UnrecoverableServerException, "#{response.code}: #{response.message}"
         end
         response
       end
@@ -496,6 +600,16 @@ module NewRelic
         ::NewRelic::Agent.logger.debug "Received response, status: #{response.code}, encoding: '#{response['content-encoding']}'"
       end
 
+      # Per protocol 17, this metric should be recorded for all error response codes
+      # that cause data to be discarded.
+      def record_error_response_supportability_metrics(response_code)
+        ::NewRelic::Agent.increment_metric("Supportability/Agent/Collector/HTTPError/#{response_code}")
+      end
+
+      def record_endpoint_attempts_supportability_metrics(endpoint)
+        ::NewRelic::Agent.increment_metric("Supportability/Agent/Collector/#{endpoint}/Attempts")
+      end
+
       # Decompresses the response from the server, if it is gzip
       # encoded, otherwise returns it verbatim
       def decompress_response(response)
@@ -518,9 +632,6 @@ module NewRelic
         zlib_version << "zlib/#{Zlib.zlib_version}" if defined?(::Zlib) && Zlib.respond_to?(:zlib_version)
         "NewRelic-RubyAgent/#{NewRelic::VERSION::STRING} #{ruby_description}#{zlib_version}"
       end
-
-      # Used to wrap errors reported to agent by the collector
-      class CollectorError < StandardError; end
     end
   end
 end