newrelic_rpm 3.12.0.288 → 9.10.2

data/lib/new_relic/agent/local_log_decorator.rb

@@ -0,0 +1,37 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    # This module contains helper methods related to decorating log messages
+    module LocalLogDecorator
+      extend self
+
+      def decorate(message)
+        return message unless decorating_enabled?
+
+        metadata = NewRelic::Agent.linking_metadata
+        formatted_metadata = " NR-LINKING|#{metadata[ENTITY_GUID_KEY]}|#{metadata[HOSTNAME_KEY]}|" \
+                             "#{metadata[TRACE_ID_KEY]}|#{metadata[SPAN_ID_KEY]}|" \
+                             "#{escape_entity_name(metadata[ENTITY_NAME_KEY])}|"
+
+        message.partition("\n").insert(1, formatted_metadata).join
+      end
+
+      private
+
+      def decorating_enabled?
+        NewRelic::Agent.config[:'application_logging.enabled'] &&
+          NewRelic::Agent::Instrumentation::Logger.enabled? &&
+          NewRelic::Agent.config[:'application_logging.local_decorating.enabled']
+      end
+
+      def escape_entity_name(entity_name)
+        return unless entity_name
+
+        URI::DEFAULT_PARSER.escape(entity_name)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_event_aggregator.rb

@@ -0,0 +1,267 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+require 'new_relic/agent/log_priority'
+require 'new_relic/agent/log_event_attributes'
+
+module NewRelic
+  module Agent
+    class LogEventAggregator < EventAggregator
+      # Per-message keys
+      LEVEL_KEY = 'level'.freeze
+      MESSAGE_KEY = 'message'.freeze
+      TIMESTAMP_KEY = 'timestamp'.freeze
+      PRIORITY_KEY = 'priority'.freeze
+
+      # Metric keys
+      LINES = 'Logging/lines'.freeze
+      DROPPED_METRIC = 'Logging/Forwarding/Dropped'.freeze
+      SEEN_METRIC = 'Supportability/Logging/Forwarding/Seen'.freeze
+      SENT_METRIC = 'Supportability/Logging/Forwarding/Sent'.freeze
+      OVERALL_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Ruby/Logger/%s'.freeze
+      METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
+      FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
+      DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
+
+      named :LogEventAggregator
+      buffer_class PrioritySampledBuffer
+
+      capacity_key :'application_logging.forwarding.max_samples_stored'
+      enabled_key :'application_logging.enabled'
+
+      # Config keys
+      OVERALL_ENABLED_KEY = :'application_logging.enabled'
+      METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
+      FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
+      DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+      LOG_LEVEL_KEY = :'application_logging.forwarding.log_level'
+      CUSTOM_ATTRIBUTES_KEY = :'application_logging.forwarding.custom_attributes'
+
+      attr_reader :attributes
+
+      def initialize(events)
+        super(events)
+        @counter_lock = Mutex.new
+        @seen = 0
+        @seen_by_severity = Hash.new(0)
+        @high_security = NewRelic::Agent.config[:high_security]
+        @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
+        @attributes = NewRelic::Agent::LogEventAttributes.new
+        register_for_done_configuring(events)
+      end
+
+      def capacity
+        @buffer.capacity
+      end
+
+      def record(formatted_message, severity)
+        return unless enabled?
+
+        severity = 'UNKNOWN' if severity.nil? || severity.empty?
+
+        if NewRelic::Agent.config[METRICS_ENABLED_KEY]
+          @counter_lock.synchronize do
+            @seen += 1
+            @seen_by_severity[severity] += 1
+          end
+        end
+
+        return if severity_too_low?(severity)
+        return if formatted_message.nil? || formatted_message.empty?
+        return unless NewRelic::Agent.config[FORWARDING_ENABLED_KEY]
+        return if @high_security
+
+        txn = NewRelic::Agent::Transaction.tl_current
+        priority = LogPriority.priority_for(txn)
+
+        if txn
+          return txn.add_log_event(create_event(priority, formatted_message, severity))
+        else
+          return @lock.synchronize do
+            @buffer.append(priority: priority) do
+              create_event(priority, formatted_message, severity)
+            end
+          end
+        end
+      rescue
+        nil
+      end
+
+      def record_batch(txn, logs)
+        # Ensure we have the same shared priority
+        priority = LogPriority.priority_for(txn)
+        logs.each do |log|
+          log.first[PRIORITY_KEY] = priority
+        end
+
+        @lock.synchronize do
+          logs.each do |log|
+            @buffer.append(event: log)
+          end
+        end
+      end
+
+      def create_event(priority, formatted_message, severity)
+        formatted_message = truncate_message(formatted_message)
+
+        event = LinkingMetadata.append_trace_linking_metadata({
+          LEVEL_KEY => severity,
+          MESSAGE_KEY => formatted_message,
+          TIMESTAMP_KEY => Process.clock_gettime(Process::CLOCK_REALTIME) * 1000
+        })
+
+        [
+          {
+            PrioritySampledBuffer::PRIORITY_KEY => priority
+          },
+          event
+        ]
+      end
+
+      def add_custom_attributes(custom_attributes)
+        attributes.add_custom_attributes(custom_attributes)
+      end
+
+      # Because our transmission format (MELT) is different than historical
+      # agent payloads, extract the munging here to keep the service focused
+      # on the general harvest + transmit instead of the format.
+      #
+      # Payload shape matches the publicly documented MELT format.
+      # https://docs.newrelic.com/docs/logs/log-api/introduction-log-api
+      #
+      # We have to keep the aggregated payloads in a separate shape, though, to
+      # work with the priority sampling buffers
+      def self.payload_to_melt_format(data)
+        common_attributes = LinkingMetadata.append_service_linking_metadata({})
+
+        # To save on unnecessary data transmission, trim the entity.type
+        # sent by classic logs-in-context
+        common_attributes.delete(ENTITY_TYPE_KEY)
+
+        common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)
+
+        _, items = data
+        payload = [{
+          common: {attributes: common_attributes},
+          logs: items.map(&:last)
+        }]
+
+        return [payload, items.size]
+      end
+
+      def harvest!
+        record_customer_metrics()
+        super
+      end
+
+      def reset!
+        @counter_lock.synchronize do
+          @seen = 0
+          @seen_by_severity.clear
+        end
+
+        super
+      end
+
+      def enabled?
+        @enabled && @instrumentation_logger_enabled
+      end
+
+      private
+
+      # We record once-per-connect metrics for enabled/disabled state at the
+      # point we consider the configuration stable (i.e. once we've gotten SSC)
+      def register_for_done_configuring(events)
+        events.subscribe(:server_source_configuration_added) do
+          @high_security = NewRelic::Agent.config[:high_security]
+
+          record_configuration_metric(OVERALL_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
+          record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
+          record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
+          record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+
+          add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
+        end
+      end
+
+      def record_configuration_metric(format, key)
+        state = NewRelic::Agent.config[key]
+        label = if !enabled?
+          'disabled'
+        else
+          state ? 'enabled' : 'disabled'
+        end
+        NewRelic::Agent.increment_metric(format % label)
+      end
+
+      def after_harvest(metadata)
+        dropped_count = metadata[:seen] - metadata[:captured]
+        note_dropped_events(metadata[:seen], dropped_count)
+        record_supportability_metrics(metadata[:seen], metadata[:captured], dropped_count)
+      end
+
+      # To avoid paying the cost of metric recording on every line, we hold
+      # these until harvest before recording them
+      def record_customer_metrics
+        return unless enabled?
+        return unless NewRelic::Agent.config[METRICS_ENABLED_KEY]
+
+        @counter_lock.synchronize do
+          return unless @seen > 0
+
+          NewRelic::Agent.increment_metric(LINES, @seen)
+          @seen_by_severity.each do |(severity, count)|
+            NewRelic::Agent.increment_metric(line_metric_name_by_severity(severity), count)
+          end
+
+          @seen = 0
+          @seen_by_severity.clear
+        end
+      end
+
+      def line_metric_name_by_severity(severity)
+        @line_metrics ||= {}
+        @line_metrics[severity] ||= "Logging/lines/#{severity}".freeze
+      end
+
+      def note_dropped_events(total_count, dropped_count)
+        if dropped_count > 0
+          NewRelic::Agent.logger.warn("Dropped #{dropped_count} log events out of #{total_count}.")
+        end
+      end
+
+      def record_supportability_metrics(total_count, captured_count, dropped_count)
+        return unless total_count > 0
+
+        NewRelic::Agent.increment_metric(DROPPED_METRIC, dropped_count)
+        NewRelic::Agent.increment_metric(SEEN_METRIC, total_count)
+        NewRelic::Agent.increment_metric(SENT_METRIC, captured_count)
+      end
+
+      def truncate_message(message)
+        return message if message.bytesize <= MAX_BYTES
+
+        message.byteslice(0...MAX_BYTES)
+      end
+
+      def configured_log_level_constant
+        format_log_level_constant(NewRelic::Agent.config[LOG_LEVEL_KEY])
+      end
+
+      def format_log_level_constant(log_level)
+        log_level.upcase.to_sym
+      end
+
+      def severity_too_low?(severity)
+        severity_constant = format_log_level_constant(severity)
+        # always record custom log levels
+        return false unless Logger::Severity.constants.include?(severity_constant)
+
+        Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_event_attributes.rb

@@ -0,0 +1,115 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    class LogEventAttributes
+      MAX_ATTRIBUTE_COUNT = 240 # limit is 255, assume we send 15
+      ATTRIBUTE_KEY_CHARACTER_LIMIT = 255
+      ATTRIBUTE_VALUE_CHARACTER_LIMIT = 4094
+
+      def add_custom_attributes(attributes)
+        return if defined?(@custom_attribute_limit_reached) && @custom_attribute_limit_reached
+
+        attributes.each do |key, value|
+          next if absent?(key) || absent?(value)
+
+          add_custom_attribute(key, value)
+        end
+      end
+
+      def custom_attributes
+        @custom_attributes ||= {}
+      end
+
+      private
+
+      class TruncationError < StandardError
+        attr_reader :attribute, :limit
+
+        def initialize(attribute, limit, msg = "Can't truncate")
+          @attribute = attribute
+          @limit = limit
+          super(msg)
+        end
+      end
+
+      class InvalidTypeError < StandardError
+        attr_reader :attribute
+
+        def initialize(attribute, msg = 'Invalid attribute type')
+          @attribute = attribute
+          super(msg)
+        end
+      end
+
+      def absent?(value)
+        value.nil? || (value.respond_to?(:empty?) && value.empty?)
+      end
+
+      def add_custom_attribute(key, value)
+        if custom_attributes.size >= MAX_ATTRIBUTE_COUNT
+          NewRelic::Agent.logger.warn(
+            'Too many custom log attributes defined. ' \
+            "Only taking the first #{MAX_ATTRIBUTE_COUNT}."
+          )
+          @custom_attribute_limit_reached = true
+          return
+        end
+
+        @custom_attributes.merge!(truncate_attributes(key_to_string(key), value))
+      end
+
+      def key_to_string(key)
+        key.is_a?(String) ? key : key.to_s
+      end
+
+      def truncate_attribute(attribute, limit)
+        case attribute
+        when Integer
+          if attribute.digits.length > limit
+            raise TruncationError.new(attribute, limit)
+          end
+        when Float
+          if attribute.to_s.length > limit
+            raise TruncationError.new(attribute, limit)
+          end
+        when String, Symbol
+          if attribute.length > limit
+            attribute = attribute.slice(0..(limit - 1))
+          end
+        when TrueClass, FalseClass
+          attribute
+        else
+          raise InvalidTypeError.new(attribute)
+        end
+
+        attribute
+      end
+
+      def truncate_attributes(key, value)
+        key = truncate_attribute(key, ATTRIBUTE_KEY_CHARACTER_LIMIT)
+        value = truncate_attribute(value, ATTRIBUTE_VALUE_CHARACTER_LIMIT)
+
+        {key => value}
+      rescue TruncationError => e
+        NewRelic::Agent.logger.warn(
+          "Dropping custom log attribute #{key} => #{value} \n" \
+          "Length exceeds character limit of #{e.limit}. " \
+          "Can't truncate: #{e.attribute}"
+        )
+
+        {}
+      rescue InvalidTypeError => e
+        NewRelic::Agent.logger.warn(
+          "Dropping custom log attribute #{key} => #{value} \n" \
+          "Invalid type of #{e.attribute.class} given. " \
+          "Can't send #{e.attribute}."
+        )
+
+        {}
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_once.rb

@@ -0,0 +1,39 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    module LogOnce
+      NUM_LOG_ONCE_KEYS = 1000
+
+      def log_once(level, key, *msgs)
+        @already_logged_lock.synchronize do
+          return if @already_logged.include?(key)
+
+          if @already_logged.size >= NUM_LOG_ONCE_KEYS && key.kind_of?(String)
+            # The reason for preventing too many keys in `logged` is for
+            # memory concerns.
+            # The reason for checking the type of the key is that we always want
+            # to allow symbols to log, since there are very few of them.
+            # The assumption here is that you would NEVER pass dynamically-created
+            # symbols, because you would never create symbols dynamically in the
+            # first place, as that would already be a memory leak in most Rubies,
+            # even if we didn't hang on to them all here.
+            return
+          end
+
+          @already_logged[key] = true
+        end
+
+        self.send(level, *msgs)
+      end
+
+      def clear_already_logged
+        @already_logged_lock.synchronize do
+          @already_logged = {}
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/log_priority.rb

@@ -0,0 +1,20 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'new_relic/agent/event_aggregator'
+
+# Stateless calculation of priority for a given log event
+module NewRelic
+  module Agent
+    module LogPriority
+      extend self
+
+      def priority_for(txn)
+        return txn.priority if txn
+
+        rand.round(NewRelic::PRIORITY_PRECISION)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/logging.rb

@@ -0,0 +1,182 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'json'
+require 'new_relic/agent/hostname'
+
+module NewRelic
+  module Agent
+    module Logging
+      # This class can be used as the formatter for an existing logger.  It
+      # decorates log messages with trace and entity metadata, and formats each
+      # log messages as a JSON object.
+      #
+      # It can be added to a Rails application like this:
+      #
+      #   require 'newrelic_rpm'
+      #
+      #   Rails.application.configure do
+      #     config.log_formatter = ::NewRelic::Agent::Logging::DecoratingFormatter.new
+      #   end
+      #
+      # @api public
+      class DecoratingFormatter < ::Logger::Formatter
+        MESSAGE_ELEMENT = 'message'
+        LOG_LEVEL_ELEMENT = 'log_level'
+        PROG_NAME_ELEMENT = 'prog_name'
+        ELEMENTS = ['app_name', 'entity_type', 'hostname', 'entity_guid', 'trace_id', 'span_id', MESSAGE_ELEMENT,
+          LOG_LEVEL_ELEMENT, PROG_NAME_ELEMENT].freeze
+        TIMESTAMP_KEY = 'timestamp'
+        MESSAGE_KEY = 'message'
+        LOG_LEVEL_KEY = 'log.level'
+        LOG_NAME_KEY = 'logger.name'
+        NEWLINE = "\n"
+
+        QUOTE = '"'
+        COLON = ':'
+        COMMA = ','
+        CLOSING_BRACE = '}'
+        REPLACEMENT_CHAR = '�'
+
+        def initialize
+          Agent.config.register_callback(:app_name) do
+            @app_name = nil
+          end
+        end
+
+        def call(severity, time, progname, msg)
+          message = +'{'
+          ELEMENTS.each do |element|
+            args = case element
+                   when MESSAGE_ELEMENT then [message, msg]
+                   when LOG_LEVEL_ELEMENT then [message, severity]
+                   when PROG_NAME_ELEMENT then [message, progname]
+                   else [message]
+            end
+
+            send("add_#{element}", *args)
+          end
+          message << COMMA
+          message << QUOTE << TIMESTAMP_KEY << QUOTE << COLON << (time.to_f * 1000).round.to_s
+          message << CLOSING_BRACE << NEWLINE
+        end
+
+        def clear_tags!
+          # No-op; just avoiding issues with act-fluent-logger-rails
+        end
+
+        private
+
+        def add_app_name(message)
+          return unless app_name
+
+          add_key_value(message, ENTITY_NAME_KEY, app_name)
+          message << COMMA
+        end
+
+        def add_entity_type(message)
+          add_key_value(message, ENTITY_TYPE_KEY, ENTITY_TYPE)
+          message << COMMA
+        end
+
+        def add_hostname(message)
+          add_key_value(message, HOSTNAME_KEY, Hostname.get)
+        end
+
+        def add_entity_guid(message)
+          return unless entity_guid = Agent.config[:entity_guid]
+
+          message << COMMA
+          add_key_value(message, ENTITY_GUID_KEY, entity_guid)
+        end
+
+        def add_trace_id(message)
+          return unless trace_id = Tracer.trace_id
+
+          message << COMMA
+          add_key_value(message, TRACE_ID_KEY, trace_id)
+        end
+
+        def add_span_id(message)
+          return unless span_id = Tracer.span_id
+
+          message << COMMA
+          add_key_value(message, SPAN_ID_KEY, span_id)
+        end
+
+        def add_message(message, msg)
+          message << COMMA
+          message << QUOTE << MESSAGE_KEY << QUOTE << COLON << escape(msg)
+          message << COMMA
+        end
+
+        def add_log_level(message, severity)
+          add_key_value(message, LOG_LEVEL_KEY, severity)
+        end
+
+        def add_prog_name(message, progname)
+          return unless progname
+
+          message << COMMA
+          add_key_value(message, LOG_NAME_KEY, progname)
+        end
+
+        def app_name
+          @app_name ||= Agent.config[:app_name][0]
+        end
+
+        def add_key_value(message, key, value)
+          message << QUOTE << key << QUOTE << COLON << QUOTE << value << QUOTE
+        end
+
+        def escape(message)
+          message = message.to_s unless message.is_a?(String)
+          unless message.encoding == Encoding::UTF_8 && message.valid_encoding?
+            message = message.encode(
+              Encoding::UTF_8,
+              invalid: :replace,
+              undef: :replace,
+              replace: REPLACEMENT_CHAR
+            )
+          end
+          message.to_json
+        end
+      end
+
+      # This logger decorates logs with trace and entity metadata, and emits log
+      # messages formatted as JSON objects.  It extends the Logger class from
+      # the Ruby standard library, and accepts the same constructor parameters.
+      #
+      # It aliases the `:info` message to overwrite the `:write` method, so it
+      # can be used in Rack applications that expect the logger to be a file-like
+      # object.
+      #
+      # It can be added to an application like this:
+      #
+      #   require 'newrelic_rpm'
+      #
+      #   config.logger = NewRelic::Agent::Logging::DecoratingLogger.new "log/application.log"
+      #
+      # @api public
+      class DecoratingLogger < (defined?(::ActiveSupport) && defined?(::ActiveSupport::Logger) ? ::ActiveSupport::Logger : ::Logger)
+        alias :write :info
+
+        # Positional and Keyword arguments are separated beginning with Ruby 2.7
+        # Signature of ::Logger constructor changes in Ruby 2.4 to have both positional and keyword args
+        # We pivot on Ruby 2.7 for widest supportability with least amount of hassle.
+        if RUBY_VERSION < '2.7.0'
+          def initialize(*args)
+            super(*args)
+            self.formatter = DecoratingFormatter.new
+          end
+        else
+          def initialize(*args, **kwargs)
+            super(*args, **kwargs)
+            self.formatter = DecoratingFormatter.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/agent/memory_logger.rb

@@ -1,13 +1,20 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 # Base class for startup logging and testing in multiverse
 
+require 'new_relic/agent/log_once'
+
 module NewRelic
   module Agent
     class MemoryLogger
+      include LogOnce
+
       def initialize
+        @already_logged_lock = Mutex.new
+        clear_already_logged
+
         @messages = []
       end
 
@@ -15,7 +22,7 @@ module NewRelic
         true
       end
 
-      attr_accessor :messages, :level
+      attr_accessor :messages, :level, :log_formatter
 
       def fatal(*msgs, &blk)
         messages << [:fatal, msgs, blk]
@@ -37,7 +44,7 @@ module NewRelic
         messages << [:debug, msgs, blk]
       end
 
-      def log_exception(level, e, backtrace_level=level)
+      def log_exception(level, e, backtrace_level = level)
         messages << [:log_exception, [level, e, backtrace_level]]
       end