newrelic_rpm 6.8.0.360 → 9.2.2

data/lib/new_relic/agent/log_event_aggregator.rb

@@ -0,0 +1,235 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+require 'new_relic/agent/log_priority'
+
+module NewRelic
+  module Agent
+    class LogEventAggregator < EventAggregator
+      # Per-message keys
+      LEVEL_KEY = 'level'.freeze
+      MESSAGE_KEY = 'message'.freeze
+      TIMESTAMP_KEY = 'timestamp'.freeze
+      PRIORITY_KEY = 'priority'.freeze
+
+      # Metric keys
+      LINES = 'Logging/lines'.freeze
+      DROPPED_METRIC = 'Logging/Forwarding/Dropped'.freeze
+      SEEN_METRIC = 'Supportability/Logging/Forwarding/Seen'.freeze
+      SENT_METRIC = 'Supportability/Logging/Forwarding/Sent'.freeze
+      OVERALL_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Ruby/Logger/%s'.freeze
+      METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
+      FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
+      DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
+
+      named :LogEventAggregator
+      buffer_class PrioritySampledBuffer
+
+      capacity_key :'application_logging.forwarding.max_samples_stored'
+      enabled_key :'application_logging.enabled'
+
+      # Config keys
+      OVERALL_ENABLED_KEY = :'application_logging.enabled'
+      METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
+      FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
+      DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+
+      def initialize(events)
+        super(events)
+        @counter_lock = Mutex.new
+        @seen = 0
+        @seen_by_severity = Hash.new(0)
+        @high_security = NewRelic::Agent.config[:high_security]
+        @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
+        register_for_done_configuring(events)
+      end
+
+      def capacity
+        @buffer.capacity
+      end
+
+      def record(formatted_message, severity)
+        return unless enabled?
+
+        severity = 'UNKNOWN' if severity.nil? || severity.empty?
+
+        if NewRelic::Agent.config[METRICS_ENABLED_KEY]
+          @counter_lock.synchronize do
+            @seen += 1
+            @seen_by_severity[severity] += 1
+          end
+        end
+
+        return if formatted_message.nil? || formatted_message.empty?
+        return unless NewRelic::Agent.config[:'application_logging.forwarding.enabled']
+        return if @high_security
+
+        txn = NewRelic::Agent::Transaction.tl_current
+        priority = LogPriority.priority_for(txn)
+
+        if txn
+          return txn.add_log_event(create_event(priority, formatted_message, severity))
+        else
+          return @lock.synchronize do
+            @buffer.append(priority: priority) do
+              create_event(priority, formatted_message, severity)
+            end
+          end
+        end
+      rescue
+        nil
+      end
+
+      def record_batch(txn, logs)
+        # Ensure we have the same shared priority
+        priority = LogPriority.priority_for(txn)
+        logs.each do |log|
+          log.first[PRIORITY_KEY] = priority
+        end
+
+        @lock.synchronize do
+          logs.each do |log|
+            @buffer.append(event: log)
+          end
+        end
+      end
+
+      def create_event(priority, formatted_message, severity)
+        formatted_message = truncate_message(formatted_message)
+
+        event = LinkingMetadata.append_trace_linking_metadata({
+          LEVEL_KEY => severity,
+          MESSAGE_KEY => formatted_message,
+          TIMESTAMP_KEY => Process.clock_gettime(Process::CLOCK_REALTIME) * 1000
+        })
+
+        [
+          {
+            PrioritySampledBuffer::PRIORITY_KEY => priority
+          },
+          event
+        ]
+      end
+
+      # Because our transmission format (MELT) is different than historical
+      # agent payloads, extract the munging here to keep the service focused
+      # on the general harvest + transmit instead of the format.
+      #
+      # Payload shape matches the publicly documented MELT format.
+      # https://docs.newrelic.com/docs/logs/log-api/introduction-log-api
+      #
+      # We have to keep the aggregated payloads in a separate shape, though, to
+      # work with the priority sampling buffers
+      def self.payload_to_melt_format(data)
+        common_attributes = LinkingMetadata.append_service_linking_metadata({})
+
+        # To save on unnecessary data transmission, trim the entity.type
+        # sent by classic logs-in-context
+        common_attributes.delete(ENTITY_TYPE_KEY)
+
+        _, items = data
+        payload = [{
+          common: {attributes: common_attributes},
+          logs: items.map(&:last)
+        }]
+
+        return [payload, items.size]
+      end
+
+      def harvest!
+        record_customer_metrics()
+        super
+      end
+
+      def reset!
+        @counter_lock.synchronize do
+          @seen = 0
+          @seen_by_severity.clear
+        end
+        super
+      end
+
+      def enabled?
+        @enabled && @instrumentation_logger_enabled
+      end
+
+      private
+
+      # We record once-per-connect metrics for enabled/disabled state at the
+      # point we consider the configuration stable (i.e. once we've gotten SSC)
+      def register_for_done_configuring(events)
+        events.subscribe(:server_source_configuration_added) do
+          @high_security = NewRelic::Agent.config[:high_security]
+
+          record_configuration_metric(OVERALL_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
+          record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
+          record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
+          record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+        end
+      end
+
+      def record_configuration_metric(format, key)
+        state = NewRelic::Agent.config[key]
+        label = if !enabled?
+          'disabled'
+        else
+          state ? 'enabled' : 'disabled'
+        end
+        NewRelic::Agent.increment_metric(format % label)
+      end
+
+      def after_harvest(metadata)
+        dropped_count = metadata[:seen] - metadata[:captured]
+        note_dropped_events(metadata[:seen], dropped_count)
+        record_supportability_metrics(metadata[:seen], metadata[:captured], dropped_count)
+      end
+
+      # To avoid paying the cost of metric recording on every line, we hold
+      # these until harvest before recording them
+      def record_customer_metrics
+        return unless enabled?
+        return unless NewRelic::Agent.config[:'application_logging.metrics.enabled']
+
+        @counter_lock.synchronize do
+          return unless @seen > 0
+
+          NewRelic::Agent.increment_metric(LINES, @seen)
+          @seen_by_severity.each do |(severity, count)|
+            NewRelic::Agent.increment_metric(line_metric_name_by_severity(severity), count)
+          end
+
+          @seen = 0
+          @seen_by_severity.clear
+        end
+      end
+
+      def line_metric_name_by_severity(severity)
+        @line_metrics ||= {}
+        @line_metrics[severity] ||= "Logging/lines/#{severity}".freeze
+      end
+
+      def note_dropped_events(total_count, dropped_count)
+        if dropped_count > 0
+          NewRelic::Agent.logger.warn("Dropped #{dropped_count} log events out of #{total_count}.")
+        end
+      end
+
+      def record_supportability_metrics(total_count, captured_count, dropped_count)
+        return unless total_count > 0
+
+        NewRelic::Agent.increment_metric(DROPPED_METRIC, dropped_count)
+        NewRelic::Agent.increment_metric(SEEN_METRIC, total_count)
+        NewRelic::Agent.increment_metric(SENT_METRIC, captured_count)
+      end
+
+      def truncate_message(message)
+        return message if message.bytesize <= MAX_BYTES
+
+        message.byteslice(0...MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_once.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 module NewRelic
   module Agent

data/lib/new_relic/agent/log_priority.rb

@@ -0,0 +1,20 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+
+# Stateless calculation of priority for a given log event
+module NewRelic
+  module Agent
+    module LogPriority
+      extend self
+
+      def priority_for(txn)
+        return txn.priority if txn
+
+        rand.round(NewRelic::PRIORITY_PRECISION)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/logging.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 require 'json'
 require 'new_relic/agent/hostname'
@@ -8,7 +8,6 @@ require 'new_relic/agent/hostname'
 module NewRelic
   module Agent
     module Logging
-
       # This class can be used as the formatter for an existing logger.  It
       # decorates log messages with trace and entity metadata, and formats each
       # log messages as a JSON object.
@@ -23,75 +22,121 @@ module NewRelic
       #
       # @api public
       class DecoratingFormatter < ::Logger::Formatter
-        TIMESTAMP_KEY = 'timestamp'.freeze
-        MESSAGE_KEY = 'message'.freeze
-        LOG_LEVEL_KEY = 'log.level'.freeze
-        LOG_NAME_KEY = 'logger.name'.freeze
-        NEWLINE = "\n".freeze
-
-        QUOTE = '"'.freeze
-        COLON = ':'.freeze
-        COMMA = ','.freeze
-        CLOSING_BRACE = '}'.freeze
+        MESSAGE_ELEMENT = 'message'
+        LOG_LEVEL_ELEMENT = 'log_level'
+        PROG_NAME_ELEMENT = 'prog_name'
+        ELEMENTS = ['app_name', 'entity_type', 'hostname', 'entity_guid', 'trace_id', 'span_id', MESSAGE_ELEMENT,
+          LOG_LEVEL_ELEMENT, PROG_NAME_ELEMENT].freeze
+        TIMESTAMP_KEY = 'timestamp'
+        MESSAGE_KEY = 'message'
+        LOG_LEVEL_KEY = 'log.level'
+        LOG_NAME_KEY = 'logger.name'
+        NEWLINE = "\n"
+
+        QUOTE = '"'
+        COLON = ':'
+        COMMA = ','
+        CLOSING_BRACE = '}'
+        REPLACEMENT_CHAR = '�'
 
         def initialize
-          Agent.config.register_callback :app_name do
+          Agent.config.register_callback(:app_name) do
             @app_name = nil
           end
         end
 
-        def call severity, time, progname, msg
-          message = '{'
-          if app_name
-            add_key_value message, ENTITY_NAME_KEY, app_name
-            message << COMMA
+        def call(severity, time, progname, msg)
+          message = +'{'
+          ELEMENTS.each do |element|
+            args = case element
+                   when MESSAGE_ELEMENT then [message, msg]
+                   when LOG_LEVEL_ELEMENT then [message, severity]
+                   when PROG_NAME_ELEMENT then [message, progname]
+                   else [message]
+            end
+
+            send("add_#{element}", *args)
           end
-          add_key_value message, ENTITY_TYPE_KEY, ENTITY_TYPE
           message << COMMA
-          add_key_value message, HOSTNAME_KEY, Hostname.get
+          message << QUOTE << TIMESTAMP_KEY << QUOTE << COLON << (time.to_f * 1000).round.to_s
+          message << CLOSING_BRACE << NEWLINE
+        end
 
-          if entity_guid = Agent.config[:entity_guid]
-            message << COMMA
-            add_key_value message, ENTITY_GUID_KEY, entity_guid
-          end
+        private
 
-          if trace_id = Tracer.trace_id
-            message << COMMA
-            add_key_value message, TRACE_ID_KEY, trace_id
-          end
-          if span_id = Tracer.span_id
-            message << COMMA
-            add_key_value message, SPAN_ID_KEY, span_id
-          end
+        def add_app_name(message)
+          return unless app_name
+
+          add_key_value(message, ENTITY_NAME_KEY, app_name)
+          message << COMMA
+        end
 
+        def add_entity_type(message)
+          add_key_value(message, ENTITY_TYPE_KEY, ENTITY_TYPE)
+          message << COMMA
+        end
+
+        def add_hostname(message)
+          add_key_value(message, HOSTNAME_KEY, Hostname.get)
+        end
+
+        def add_entity_guid(message)
+          return unless entity_guid = Agent.config[:entity_guid]
+
+          message << COMMA
+          add_key_value(message, ENTITY_GUID_KEY, entity_guid)
+        end
+
+        def add_trace_id(message)
+          return unless trace_id = Tracer.trace_id
+
+          message << COMMA
+          add_key_value(message, TRACE_ID_KEY, trace_id)
+        end
+
+        def add_span_id(message)
+          return unless span_id = Tracer.span_id
+
+          message << COMMA
+          add_key_value(message, SPAN_ID_KEY, span_id)
+        end
+
+        def add_message(message, msg)
           message << COMMA
           message << QUOTE << MESSAGE_KEY << QUOTE << COLON << escape(msg)
           message << COMMA
-          add_key_value message, LOG_LEVEL_KEY, severity
-          if progname
-            message << COMMA
-            add_key_value message, LOG_NAME_KEY, progname
-          end
+        end
+
+        def add_log_level(message, severity)
+          add_key_value(message, LOG_LEVEL_KEY, severity)
+        end
+
+        def add_prog_name(message, progname)
+          return unless progname
 
           message << COMMA
-          message << QUOTE << TIMESTAMP_KEY << QUOTE << COLON << time.to_f.to_s
-          message << CLOSING_BRACE << NEWLINE
+          add_key_value(message, LOG_NAME_KEY, progname)
         end
 
         def app_name
           @app_name ||= Agent.config[:app_name][0]
         end
 
-        def add_key_value message, key, value
+        def add_key_value(message, key, value)
           message << QUOTE << key << QUOTE << COLON << QUOTE << value << QUOTE
         end
 
-        def escape message
-          if String === message
-            message.to_json
-          else
-            message.inspect.to_json
+        def escape(message)
+          message = message.to_s unless message.is_a?(String)
+          unless message.encoding == Encoding::UTF_8 && message.valid_encoding?
+            message = message.encode(
+              Encoding::UTF_8,
+              invalid: :replace,
+              undef: :replace,
+              replace: REPLACEMENT_CHAR
+            )
           end
+          message.to_json
         end
 
         def clear_tags!
@@ -99,7 +144,6 @@ module NewRelic
         end
       end
 
-
       # This logger decorates logs with trace and entity metadata, and emits log
       # messages formatted as JSON objects.  It extends the Logger class from
       # the Ruby standard library, and accepts the same constructor parameters.
@@ -116,12 +160,21 @@ module NewRelic
       #
       # @api public
       class DecoratingLogger < (defined?(::ActiveSupport) && defined?(::ActiveSupport::Logger) ? ::ActiveSupport::Logger : ::Logger)
-
         alias :write :info
 
-        def initialize *args
-          super
-          self.formatter = DecoratingFormatter.new
+        # Positional and Keyword arguments are separated beginning with Ruby 2.7
+        # Signature of ::Logger constructor changes in Ruby 2.4 to have both positional and keyword args
+        # We pivot on Ruby 2.7 for widest supportability with least amount of hassle.
+        if RUBY_VERSION < '2.7.0'
+          def initialize(*args)
+            super(*args)
+            self.formatter = DecoratingFormatter.new
+          end
+        else
+          def initialize(*args, **kwargs)
+            super(*args, **kwargs)
+            self.formatter = DecoratingFormatter.new
+          end
         end
       end
     end

data/lib/new_relic/agent/memory_logger.rb

@@ -1,6 +1,6 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 # Base class for startup logging and testing in multiverse
 
@@ -44,7 +44,7 @@ module NewRelic
         messages << [:debug, msgs, blk]
       end
 
-      def log_exception(level, e, backtrace_level=level)
+      def log_exception(level, e, backtrace_level = level)
         messages << [:log_exception, [level, e, backtrace_level]]
       end