newrelic_rpm 8.2.0 → 9.0.0

data/lib/new_relic/agent/log_event_aggregator.rb

@@ -0,0 +1,235 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+require 'new_relic/agent/log_priority'
+
+module NewRelic
+  module Agent
+    class LogEventAggregator < EventAggregator
+      # Per-message keys
+      LEVEL_KEY = "level".freeze
+      MESSAGE_KEY = "message".freeze
+      TIMESTAMP_KEY = "timestamp".freeze
+      PRIORITY_KEY = "priority".freeze
+
+      # Metric keys
+      LINES = "Logging/lines".freeze
+      DROPPED_METRIC = "Logging/Forwarding/Dropped".freeze
+      SEEN_METRIC = "Supportability/Logging/Forwarding/Seen".freeze
+      SENT_METRIC = "Supportability/Logging/Forwarding/Sent".freeze
+      OVERALL_SUPPORTABILITY_FORMAT = "Supportability/Logging/Ruby/Logger/%s".freeze
+      METRICS_SUPPORTABILITY_FORMAT = "Supportability/Logging/Metrics/Ruby/%s".freeze
+      FORWARDING_SUPPORTABILITY_FORMAT = "Supportability/Logging/Forwarding/Ruby/%s".freeze
+      DECORATING_SUPPORTABILITY_FORMAT = "Supportability/Logging/LocalDecorating/Ruby/%s".freeze
+      MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
+
+      named :LogEventAggregator
+      buffer_class PrioritySampledBuffer
+
+      capacity_key :'application_logging.forwarding.max_samples_stored'
+      enabled_key :'application_logging.enabled'
+
+      # Config keys
+      OVERALL_ENABLED_KEY = :'application_logging.enabled'
+      METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
+      FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
+      DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+
+      def initialize(events)
+        super(events)
+        @counter_lock = Mutex.new
+        @seen = 0
+        @seen_by_severity = Hash.new(0)
+        @high_security = NewRelic::Agent.config[:high_security]
+        @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
+        register_for_done_configuring(events)
+      end
+
+      def capacity
+        @buffer.capacity
+      end
+
+      def record(formatted_message, severity)
+        return unless enabled?
+
+        severity = "UNKNOWN" if severity.nil? || severity.empty?
+
+        if NewRelic::Agent.config[METRICS_ENABLED_KEY]
+          @counter_lock.synchronize do
+            @seen += 1
+            @seen_by_severity[severity] += 1
+          end
+        end
+
+        return if formatted_message.nil? || formatted_message.empty?
+        return unless NewRelic::Agent.config[:'application_logging.forwarding.enabled']
+        return if @high_security
+
+        txn = NewRelic::Agent::Transaction.tl_current
+        priority = LogPriority.priority_for(txn)
+
+        if txn
+          return txn.add_log_event(create_event(priority, formatted_message, severity))
+        else
+          return @lock.synchronize do
+            @buffer.append(priority: priority) do
+              create_event(priority, formatted_message, severity)
+            end
+          end
+        end
+      rescue
+        nil
+      end
+
+      def record_batch(txn, logs)
+        # Ensure we have the same shared priority
+        priority = LogPriority.priority_for(txn)
+        logs.each do |log|
+          log.first[PRIORITY_KEY] = priority
+        end
+
+        @lock.synchronize do
+          logs.each do |log|
+            @buffer.append(event: log)
+          end
+        end
+      end
+
+      def create_event(priority, formatted_message, severity)
+        formatted_message = truncate_message(formatted_message)
+
+        event = LinkingMetadata.append_trace_linking_metadata({
+          LEVEL_KEY => severity,
+          MESSAGE_KEY => formatted_message,
+          TIMESTAMP_KEY => Process.clock_gettime(Process::CLOCK_REALTIME) * 1000
+        })
+
+        [
+          {
+            PrioritySampledBuffer::PRIORITY_KEY => priority
+          },
+          event
+        ]
+      end
+
+      # Because our transmission format (MELT) is different than historical
+      # agent payloads, extract the munging here to keep the service focused
+      # on the general harvest + transmit instead of the format.
+      #
+      # Payload shape matches the publicly documented MELT format.
+      # https://docs.newrelic.com/docs/logs/log-api/introduction-log-api
+      #
+      # We have to keep the aggregated payloads in a separate shape, though, to
+      # work with the priority sampling buffers
+      def self.payload_to_melt_format(data)
+        common_attributes = LinkingMetadata.append_service_linking_metadata({})
+
+        # To save on unnecessary data transmission, trim the entity.type
+        # sent by classic logs-in-context
+        common_attributes.delete(ENTITY_TYPE_KEY)
+
+        _, items = data
+        payload = [{
+          common: {attributes: common_attributes},
+          logs: items.map(&:last)
+        }]
+
+        return [payload, items.size]
+      end
+
+      def harvest!
+        record_customer_metrics()
+        super
+      end
+
+      def reset!
+        @counter_lock.synchronize do
+          @seen = 0
+          @seen_by_severity.clear
+        end
+        super
+      end
+
+      def enabled?
+        @enabled && @instrumentation_logger_enabled
+      end
+
+      private
+
+      # We record once-per-connect metrics for enabled/disabled state at the
+      # point we consider the configuration stable (i.e. once we've gotten SSC)
+      def register_for_done_configuring(events)
+        events.subscribe(:server_source_configuration_added) do
+          @high_security = NewRelic::Agent.config[:high_security]
+
+          record_configuration_metric(OVERALL_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
+          record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
+          record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
+          record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+        end
+      end
+
+      def record_configuration_metric(format, key)
+        state = NewRelic::Agent.config[key]
+        label = if !enabled?
+          "disabled"
+        else
+          state ? "enabled" : "disabled"
+        end
+        NewRelic::Agent.increment_metric(format % label)
+      end
+
+      def after_harvest(metadata)
+        dropped_count = metadata[:seen] - metadata[:captured]
+        note_dropped_events(metadata[:seen], dropped_count)
+        record_supportability_metrics(metadata[:seen], metadata[:captured], dropped_count)
+      end
+
+      # To avoid paying the cost of metric recording on every line, we hold
+      # these until harvest before recording them
+      def record_customer_metrics
+        return unless enabled?
+        return unless NewRelic::Agent.config[:'application_logging.metrics.enabled']
+
+        @counter_lock.synchronize do
+          return unless @seen > 0
+
+          NewRelic::Agent.increment_metric(LINES, @seen)
+          @seen_by_severity.each do |(severity, count)|
+            NewRelic::Agent.increment_metric(line_metric_name_by_severity(severity), count)
+          end
+
+          @seen = 0
+          @seen_by_severity.clear
+        end
+      end
+
+      def line_metric_name_by_severity(severity)
+        @line_metrics ||= {}
+        @line_metrics[severity] ||= "Logging/lines/#{severity}".freeze
+      end
+
+      def note_dropped_events(total_count, dropped_count)
+        if dropped_count > 0
+          NewRelic::Agent.logger.warn("Dropped #{dropped_count} log events out of #{total_count}.")
+        end
+      end
+
+      def record_supportability_metrics(total_count, captured_count, dropped_count)
+        return unless total_count > 0
+
+        NewRelic::Agent.increment_metric(DROPPED_METRIC, dropped_count)
+        NewRelic::Agent.increment_metric(SEEN_METRIC, total_count)
+        NewRelic::Agent.increment_metric(SENT_METRIC, captured_count)
+      end
+
+      def truncate_message(message)
+        return message if message.bytesize <= MAX_BYTES
+
+        message.byteslice(0...MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_once.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 module NewRelic
   module Agent

data/lib/new_relic/agent/log_priority.rb

@@ -0,0 +1,20 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+
+# Stateless calculation of priority for a given log event
+module NewRelic
+  module Agent
+    module LogPriority
+      extend self
+
+      def priority_for(txn)
+        return txn.priority if txn
+
+        rand.round(NewRelic::PRIORITY_PRECISION)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/logging.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 require 'json'
 require 'new_relic/agent/hostname'
@@ -8,7 +8,6 @@ require 'new_relic/agent/hostname'
 module NewRelic
   module Agent
     module Logging
-
       # This class can be used as the formatter for an existing logger.  It
       # decorates log messages with trace and entity metadata, and formats each
       # log messages as a JSON object.
@@ -36,42 +35,42 @@ module NewRelic
         REPLACEMENT_CHAR = '�'
 
         def initialize
-          Agent.config.register_callback :app_name do
+          Agent.config.register_callback(:app_name) do
             @app_name = nil
           end
         end
 
-        def call severity, time, progname, msg
-          message = '{'
+        def call(severity, time, progname, msg)
+          message = String.new('{')
           if app_name
-            add_key_value message, ENTITY_NAME_KEY, app_name
+            add_key_value(message, ENTITY_NAME_KEY, app_name)
             message << COMMA
           end
-          add_key_value message, ENTITY_TYPE_KEY, ENTITY_TYPE
+          add_key_value(message, ENTITY_TYPE_KEY, ENTITY_TYPE)
           message << COMMA
-          add_key_value message, HOSTNAME_KEY, Hostname.get
+          add_key_value(message, HOSTNAME_KEY, Hostname.get)
 
           if entity_guid = Agent.config[:entity_guid]
             message << COMMA
-            add_key_value message, ENTITY_GUID_KEY, entity_guid
+            add_key_value(message, ENTITY_GUID_KEY, entity_guid)
           end
 
           if trace_id = Tracer.trace_id
             message << COMMA
-            add_key_value message, TRACE_ID_KEY, trace_id
+            add_key_value(message, TRACE_ID_KEY, trace_id)
           end
           if span_id = Tracer.span_id
             message << COMMA
-            add_key_value message, SPAN_ID_KEY, span_id
+            add_key_value(message, SPAN_ID_KEY, span_id)
           end
 
           message << COMMA
           message << QUOTE << MESSAGE_KEY << QUOTE << COLON << escape(msg)
           message << COMMA
-          add_key_value message, LOG_LEVEL_KEY, severity
+          add_key_value(message, LOG_LEVEL_KEY, severity)
           if progname
             message << COMMA
-            add_key_value message, LOG_NAME_KEY, progname
+            add_key_value(message, LOG_NAME_KEY, progname)
           end
 
           message << COMMA
@@ -83,13 +82,20 @@ module NewRelic
           @app_name ||= Agent.config[:app_name][0]
         end
 
-        def add_key_value message, key, value
+        def add_key_value(message, key, value)
           message << QUOTE << key << QUOTE << COLON << QUOTE << value << QUOTE
         end
 
         def escape(message)
           message = message.to_s unless message.is_a?(String)
-          message = message.scrub(REPLACEMENT_CHAR) unless message.valid_encoding?
+          unless message.encoding == Encoding::UTF_8 && message.valid_encoding?
+            message = message.encode(
+              Encoding::UTF_8,
+              invalid: :replace,
+              undef: :replace,
+              replace: REPLACEMENT_CHAR
+            )
+          end
           message.to_json
         end
 
@@ -98,7 +104,6 @@ module NewRelic
         end
       end
 
-
       # This logger decorates logs with trace and entity metadata, and emits log
       # messages formatted as JSON objects.  It extends the Logger class from
       # the Ruby standard library, and accepts the same constructor parameters.
@@ -115,7 +120,6 @@ module NewRelic
       #
       # @api public
       class DecoratingLogger < (defined?(::ActiveSupport) && defined?(::ActiveSupport::Logger) ? ::ActiveSupport::Logger : ::Logger)
-
         alias :write :info
 
         # Positional and Keyword arguments are separated beginning with Ruby 2.7

data/lib/new_relic/agent/memory_logger.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 # Base class for startup logging and testing in multiverse
 
@@ -44,7 +44,7 @@ module NewRelic
         messages << [:debug, msgs, blk]
       end
 
-      def log_exception(level, e, backtrace_level=level)
+      def log_exception(level, e, backtrace_level = level)
         messages << [:log_exception, [level, e, backtrace_level]]
       end
 

data/lib/new_relic/agent/messaging.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
@@ -18,8 +17,8 @@ module NewRelic
       RABBITMQ_TRANSPORT_TYPE = "RabbitMQ"
 
       ATTR_DESTINATION = AttributeFilter::DST_TRANSACTION_EVENTS |
-                         AttributeFilter::DST_TRANSACTION_TRACER |
-                         AttributeFilter::DST_ERROR_COLLECTOR
+        AttributeFilter::DST_TRANSACTION_TRACER |
+        AttributeFilter::DST_ERROR_COLLECTOR
 
       # Start a MessageBroker segment configured to trace a messaging action.
       # Finishing this segment will handle timing and recording of the proper
@@ -53,12 +52,12 @@ module NewRelic
       # @api public
       #
       def start_message_broker_segment(action: nil,
-                                       library: nil,
-                                       destination_type: nil,
-                                       destination_name: nil,
-                                       headers: nil,
-                                       parameters: nil,
-                                       start_time: nil)
+        library: nil,
+        destination_type: nil,
+        destination_name: nil,
+        headers: nil,
+        parameters: nil,
+        start_time: nil)
 
         Tracer.start_message_broker_segment(
           action: action,
@@ -111,38 +110,39 @@ module NewRelic
       # @api public
       #
       def wrap_message_broker_consume_transaction(library:,
-                                                  destination_type:,
-                                                  destination_name:,
-                                                  headers: nil,
-                                                  routing_key: nil,
-                                                  queue_name: nil,
-                                                  exchange_type: nil,
-                                                  reply_to: nil,
-                                                  correlation_id: nil)
+        destination_type:,
+        destination_name:,
+        headers: nil,
+        routing_key: nil,
+        queue_name: nil,
+        exchange_type: nil,
+        reply_to: nil,
+        correlation_id: nil)
 
         state = Tracer.state
         return yield if state.current_transaction
+
         txn = nil
 
         begin
-          txn_name = transaction_name library, destination_type, destination_name
+          txn_name = transaction_name(library, destination_type, destination_name)
 
-          txn = Tracer.start_transaction name: txn_name, category: :message
+          txn = Tracer.start_transaction(name: txn_name, category: :message)
 
           if headers
-            txn.distributed_tracer.consume_message_headers headers, state, RABBITMQ_TRANSPORT_TYPE
+            txn.distributed_tracer.consume_message_headers(headers, state, RABBITMQ_TRANSPORT_TYPE)
             CrossAppTracing.reject_messaging_cat_headers(headers).each do |k, v|
-              txn.add_agent_attribute :"message.headers.#{k}", v, AttributeFilter::DST_NONE unless v.nil?
+              txn.add_agent_attribute(:"message.headers.#{k}", v, AttributeFilter::DST_NONE) unless v.nil?
             end
           end
 
-          txn.add_agent_attribute :'message.routingKey', routing_key, ATTR_DESTINATION if routing_key
-          txn.add_agent_attribute :'message.exchangeType', exchange_type, AttributeFilter::DST_NONE if exchange_type
-          txn.add_agent_attribute :'message.correlationId', correlation_id, AttributeFilter::DST_NONE if correlation_id
-          txn.add_agent_attribute :'message.queueName', queue_name, ATTR_DESTINATION if queue_name
-          txn.add_agent_attribute :'message.replyTo', reply_to, AttributeFilter::DST_NONE if reply_to
+          txn.add_agent_attribute(:'message.routingKey', routing_key, ATTR_DESTINATION) if routing_key
+          txn.add_agent_attribute(:'message.exchangeType', exchange_type, AttributeFilter::DST_NONE) if exchange_type
+          txn.add_agent_attribute(:'message.correlationId', correlation_id, AttributeFilter::DST_NONE) if correlation_id
+          txn.add_agent_attribute(:'message.queueName', queue_name, ATTR_DESTINATION) if queue_name
+          txn.add_agent_attribute(:'message.replyTo', reply_to, AttributeFilter::DST_NONE) if reply_to
         rescue => e
-          NewRelic::Agent.logger.error "Error starting Message Broker consume transaction", e
+          NewRelic::Agent.logger.error("Error starting Message Broker consume transaction", e)
         end
 
         yield
@@ -150,7 +150,7 @@ module NewRelic
         begin
           txn.finish if txn
         rescue => e
-          NewRelic::Agent.logger.error "Error stopping Message Broker consume transaction", e
+          NewRelic::Agent.logger.error("Error stopping Message Broker consume transaction", e)
         end
       end
 
@@ -181,12 +181,12 @@ module NewRelic
       # @api public
       #
       def start_amqp_publish_segment(library:,
-                                     destination_name:,
-                                     headers: nil,
-                                     routing_key: nil,
-                                     reply_to: nil,
-                                     correlation_id: nil,
-                                     exchange_type: nil)
+        destination_name:,
+        headers: nil,
+        routing_key: nil,
+        reply_to: nil,
+        correlation_id: nil,
+        exchange_type: nil)
 
         raise ArgumentError, 'missing required argument: headers' if headers.nil? && CrossAppTracing.cross_app_enabled?
 
@@ -240,12 +240,12 @@ module NewRelic
       # @api public
       #
       def start_amqp_consume_segment(library:,
-                                     destination_name:,
-                                     delivery_info:,
-                                     message_properties:,
-                                     exchange_type: nil,
-                                     queue_name: nil,
-                                     start_time: nil)
+        destination_name:,
+        delivery_info:,
+        message_properties:,
+        exchange_type: nil,
+        queue_name: nil,
+        start_time: nil)
 
         segment = Tracer.start_message_broker_segment(
           action: :consume,
@@ -258,8 +258,8 @@ module NewRelic
 
         if segment_parameters_enabled?
           if message_properties[:headers] && !message_properties[:headers].empty?
-            non_cat_headers = CrossAppTracing.reject_messaging_cat_headers message_properties[:headers]
-            non_synth_headers = SyntheticsMonitor.reject_messaging_synthetics_header non_cat_headers
+            non_cat_headers = CrossAppTracing.reject_messaging_cat_headers(message_properties[:headers])
+            non_synth_headers = SyntheticsMonitor.reject_messaging_synthetics_header(non_cat_headers)
             segment.params[:headers] = non_synth_headers unless non_synth_headers.empty?
           end
 
@@ -286,7 +286,7 @@ module NewRelic
       # @param message_properties [Hash] Metadata about the message and opaque
       #   application-level data (optional)
       #
-      # @param exchange_type [Symbol] Type of AMQP exchange the message was recieved
+      # @param exchange_type [Symbol] Type of AMQP exchange the message was received
       #   from (see NewRelic::Agent::Transaction::MessageBrokerSegment::DESTINATION_TYPES)
       #
       # @param queue_name [String] name of the AMQP queue on which the message was
@@ -300,24 +300,23 @@ module NewRelic
       #
       # @api public
       #
-      def wrap_amqp_consume_transaction library: nil,
-                                        destination_name: nil,
-                                        delivery_info: nil,
-                                        message_properties: nil,
-                                        exchange_type: nil,
-                                        queue_name: nil,
-                                        &block
-
-        wrap_message_broker_consume_transaction library: library,
-                                                destination_type: :exchange,
-                                                destination_name: Instrumentation::Bunny.exchange_name(destination_name),
-                                                routing_key: delivery_info[:routing_key],
-                                                reply_to: message_properties[:reply_to],
-                                                queue_name: queue_name,
-                                                exchange_type: exchange_type,
-                                                headers: message_properties[:headers],
-                                                correlation_id: message_properties[:correlation_id],
-                                                &block
+      def wrap_amqp_consume_transaction(library: nil,
+        destination_name: nil,
+        delivery_info: nil,
+        message_properties: nil,
+        exchange_type: nil,
+        queue_name: nil,
+        &block)
+
+        wrap_message_broker_consume_transaction(library: library,
+          destination_type: :exchange,
+          destination_name: Instrumentation::Bunny.exchange_name(destination_name),
+          routing_key: delivery_info[:routing_key],
+          reply_to: message_properties[:reply_to],
+          queue_name: queue_name,
+          exchange_type: exchange_type,
+          headers: message_properties[:headers],
+          correlation_id: message_properties[:correlation_id], &block)
       end
 
       private
@@ -326,7 +325,7 @@ module NewRelic
         NewRelic::Agent.config[:'message_tracer.segment_parameters.enabled']
       end
 
-      def transaction_name library, destination_type, destination_name
+      def transaction_name(library, destination_type, destination_name)
         transaction_name = Transaction::MESSAGE_PREFIX + library
         transaction_name << Transaction::MessageBrokerSegment::SLASH
         transaction_name << Transaction::MessageBrokerSegment::TYPES[destination_type]
@@ -352,7 +351,6 @@ module NewRelic
 
         transaction_name
       end
-
     end
   end
 end