newrelic_rpm 9.7.0 → 9.16.1

data/lib/new_relic/agent/log_event_aggregator.rb

@@ -20,10 +20,12 @@ module NewRelic
       DROPPED_METRIC = 'Logging/Forwarding/Dropped'.freeze
       SEEN_METRIC = 'Supportability/Logging/Forwarding/Seen'.freeze
       SENT_METRIC = 'Supportability/Logging/Forwarding/Sent'.freeze
-      OVERALL_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Ruby/Logger/%s'.freeze
+      LOGGER_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Ruby/Logger/%s'.freeze
+      LOGSTASHER_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Ruby/LogStasher/%s'.freeze
       METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
       FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
       DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      LABELS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Labels/Ruby/%s'.freeze
       MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
 
       named :LogEventAggregator
@@ -37,6 +39,7 @@ module NewRelic
       METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
       FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
       DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+      LABELS_ENABLED_KEY = :'application_logging.forwarding.labels.enabled'
       LOG_LEVEL_KEY = :'application_logging.forwarding.log_level'
       CUSTOM_ATTRIBUTES_KEY = :'application_logging.forwarding.custom_attributes'
 
@@ -50,6 +53,7 @@ module NewRelic
         @high_security = NewRelic::Agent.config[:high_security]
         @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
         @attributes = NewRelic::Agent::LogEventAttributes.new
+
         register_for_done_configuring(events)
       end
 
@@ -58,38 +62,71 @@ module NewRelic
       end
 
       def record(formatted_message, severity)
-        return unless enabled?
+        return unless logger_enabled?
 
         severity = 'UNKNOWN' if severity.nil? || severity.empty?
+        increment_event_counters(severity)
+
+        return if formatted_message.nil? || formatted_message.empty?
+        return unless monitoring_conditions_met?(severity)
+
+        txn = NewRelic::Agent::Transaction.tl_current
+        priority = LogPriority.priority_for(txn)
+
+        return txn.add_log_event(create_event(priority, formatted_message, severity)) if txn
 
-        if NewRelic::Agent.config[METRICS_ENABLED_KEY]
-          @counter_lock.synchronize do
-            @seen += 1
-            @seen_by_severity[severity] += 1
+        @lock.synchronize do
+          @buffer.append(priority: priority) do
+            create_event(priority, formatted_message, severity)
           end
         end
+      rescue
+        nil
+      end
 
-        return if severity_too_low?(severity)
-        return if formatted_message.nil? || formatted_message.empty?
-        return unless NewRelic::Agent.config[FORWARDING_ENABLED_KEY]
-        return if @high_security
+      def record_logstasher_event(log)
+        return unless logstasher_enabled?
+
+        # LogStasher logs do not inherently include a message key, so most logs are recorded.
+        # But when the key exists, we should not record the log if the message value is nil or empty.
+        return if log.key?('message') && (log['message'].nil? || log['message'].empty?)
+
+        severity = determine_severity(log)
+        increment_event_counters(severity)
+
+        return unless monitoring_conditions_met?(severity)
 
         txn = NewRelic::Agent::Transaction.tl_current
         priority = LogPriority.priority_for(txn)
 
-        if txn
-          return txn.add_log_event(create_event(priority, formatted_message, severity))
-        else
-          return @lock.synchronize do
-            @buffer.append(priority: priority) do
-              create_event(priority, formatted_message, severity)
-            end
+        return txn.add_log_event(create_logstasher_event(priority, severity, log)) if txn
+
+        @lock.synchronize do
+          @buffer.append(priority: priority) do
+            create_logstasher_event(priority, severity, log)
           end
         end
       rescue
         nil
       end
 
+      def monitoring_conditions_met?(severity)
+        !severity_too_low?(severity) && NewRelic::Agent.config[FORWARDING_ENABLED_KEY] && !@high_security
+      end
+
+      def determine_severity(log)
+        log['level'] ? log['level'].to_s.upcase : 'UNKNOWN'
+      end
+
+      def increment_event_counters(severity)
+        return unless NewRelic::Agent.config[METRICS_ENABLED_KEY]
+
+        @counter_lock.synchronize do
+          @seen += 1
+          @seen_by_severity[severity] += 1
+        end
+      end
+
       def record_batch(txn, logs)
         # Ensure we have the same shared priority
         priority = LogPriority.priority_for(txn)
@@ -104,15 +141,17 @@ module NewRelic
         end
       end
 
-      def create_event(priority, formatted_message, severity)
-        formatted_message = truncate_message(formatted_message)
-
-        event = LinkingMetadata.append_trace_linking_metadata({
+      def add_event_metadata(formatted_message, severity)
+        metadata = {
           LEVEL_KEY => severity,
-          MESSAGE_KEY => formatted_message,
           TIMESTAMP_KEY => Process.clock_gettime(Process::CLOCK_REALTIME) * 1000
-        })
+        }
+        metadata[MESSAGE_KEY] = formatted_message unless formatted_message.nil?
+
+        LinkingMetadata.append_trace_linking_metadata(metadata)
+      end
 
+      def create_prioritized_event(priority, event)
         [
           {
             PrioritySampledBuffer::PRIORITY_KEY => priority
@@ -121,10 +160,39 @@ module NewRelic
         ]
       end
 
+      def create_event(priority, formatted_message, severity)
+        formatted_message = truncate_message(formatted_message)
+        event = add_event_metadata(formatted_message, severity)
+
+        create_prioritized_event(priority, event)
+      end
+
+      def create_logstasher_event(priority, severity, log)
+        formatted_message = log['message'] ? truncate_message(log['message']) : nil
+        event = add_event_metadata(formatted_message, severity)
+        add_logstasher_event_attributes(event, log)
+
+        create_prioritized_event(priority, event)
+      end
+
+      def add_logstasher_event_attributes(event, log)
+        log_copy = log.dup
+        # Delete previously reported attributes
+        log_copy.delete('message')
+        log_copy.delete('level')
+        log_copy.delete('@timestamp')
+
+        event['attributes'] = log_copy
+      end
+
       def add_custom_attributes(custom_attributes)
         attributes.add_custom_attributes(custom_attributes)
       end
 
+      def labels
+        @labels ||= create_labels
+      end
+
       # Because our transmission format (MELT) is different than historical
       # agent payloads, extract the munging here to keep the service focused
       # on the general harvest + transmit instead of the format.
@@ -140,8 +208,9 @@ module NewRelic
         # To save on unnecessary data transmission, trim the entity.type
         # sent by classic logs-in-context
         common_attributes.delete(ENTITY_TYPE_KEY)
-
-        common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)
+        aggregator = NewRelic::Agent.agent.log_event_aggregator
+        common_attributes.merge!(aggregator.attributes.custom_attributes)
+        common_attributes.merge!(aggregator.labels)
 
         _, items = data
         payload = [{
@@ -166,10 +235,14 @@ module NewRelic
         super
       end
 
-      def enabled?
+      def logger_enabled?
         @enabled && @instrumentation_logger_enabled
       end
 
+      def logstasher_enabled?
+        @enabled && NewRelic::Agent::Instrumentation::LogStasher.enabled?
+      end
+
       private
 
       # We record once-per-connect metrics for enabled/disabled state at the
@@ -177,11 +250,12 @@ module NewRelic
       def register_for_done_configuring(events)
         events.subscribe(:server_source_configuration_added) do
           @high_security = NewRelic::Agent.config[:high_security]
-
-          record_configuration_metric(OVERALL_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
+          record_configuration_metric(LOGGER_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
+          record_configuration_metric(LOGSTASHER_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
           record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
           record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
           record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+          record_configuration_metric(LABELS_SUPPORTABILITY_FORMAT, LABELS_ENABLED_KEY)
 
           add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
         end
@@ -247,21 +321,6 @@ module NewRelic
         message.byteslice(0...MAX_BYTES)
       end
 
-      def minimum_log_level
-        if Logger::Severity.constants.include?(configured_log_level_constant)
-          configured_log_level_constant
-        else
-          NewRelic::Agent.logger.log_once(
-            :error,
-            'Invalid application_logging.forwarding.log_level ' \
-            "'#{NewRelic::Agent.config[LOG_LEVEL_KEY]}' specified! " \
-            "Must be one of #{Logger::Severity.constants.join('|')}. " \
-            "Using default level of 'debug'"
-          )
-          :DEBUG
-        end
-      end
-
       def configured_log_level_constant
         format_log_level_constant(NewRelic::Agent.config[LOG_LEVEL_KEY])
       end
@@ -275,7 +334,24 @@ module NewRelic
         # always record custom log levels
         return false unless Logger::Severity.constants.include?(severity_constant)
 
-        Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(minimum_log_level)
+        Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
+      end
+
+      def create_labels
+        return NewRelic::EMPTY_HASH unless NewRelic::Agent.config[LABELS_ENABLED_KEY]
+
+        downcased_exclusions = NewRelic::Agent.config[:'application_logging.forwarding.labels.exclude'].map(&:downcase)
+        log_labels = {}
+
+        NewRelic::Agent.config.parsed_labels.each do |parsed_label|
+          next if downcased_exclusions.include?(parsed_label['label_type'].downcase)
+
+          # labels are referred to as tags in the UI, so prefix the
+          # label-related attributes with 'tags.*'
+          log_labels["tags.#{parsed_label['label_type']}"] = parsed_label['label_value']
+        end
+
+        log_labels
       end
     end
   end

data/lib/new_relic/agent/messaging.rb

@@ -117,7 +117,8 @@ module NewRelic
         queue_name: nil,
         exchange_type: nil,
         reply_to: nil,
-        correlation_id: nil)
+        correlation_id: nil,
+        action: nil)
 
         state = Tracer.state
         return yield if state.current_transaction
@@ -125,12 +126,12 @@ module NewRelic
         txn = nil
 
         begin
-          txn_name = transaction_name(library, destination_type, destination_name)
+          txn_name = transaction_name(library, destination_type, destination_name, action)
 
           txn = Tracer.start_transaction(name: txn_name, category: :message)
-
           if headers
-            txn.distributed_tracer.consume_message_headers(headers, state, RABBITMQ_TRANSPORT_TYPE)
+            NewRelic::Agent::DistributedTracing::accept_distributed_trace_headers(headers, library) # to handle the new w3c headers
+            txn.distributed_tracer.consume_message_headers(headers, state, library) # to do the expected old things
             CrossAppTracing.reject_messaging_cat_headers(headers).each do |k, v|
               txn.add_agent_attribute(:"message.headers.#{k}", v, AttributeFilter::DST_NONE) unless v.nil?
             end
@@ -327,12 +328,17 @@ module NewRelic
         NewRelic::Agent.config[:'message_tracer.segment_parameters.enabled']
       end
 
-      def transaction_name(library, destination_type, destination_name)
+      def transaction_name(library, destination_type, destination_name, action = nil)
         transaction_name = Transaction::MESSAGE_PREFIX + library
         transaction_name << NewRelic::SLASH
         transaction_name << Transaction::MessageBrokerSegment::TYPES[destination_type]
         transaction_name << NewRelic::SLASH
 
+        if action == :consume
+          transaction_name << 'Consume'
+          transaction_name << NewRelic::SLASH
+        end
+
         case destination_type
         when :queue
           transaction_name << Transaction::MessageBrokerSegment::NAMED

data/lib/new_relic/agent/new_relic_service.rb

@@ -143,6 +143,9 @@ module NewRelic
       end
 
       def metric_data(stats_hash)
+        # let the serverless handler handle serialization
+        return NewRelic::Agent.agent.serverless_handler.metric_data(stats_hash) if NewRelic::Agent.agent.serverless?
+
         timeslice_start = stats_hash.started_at
         timeslice_end = stats_hash.harvested_at || Process.clock_gettime(Process::CLOCK_REALTIME)
         metric_data_array = build_metric_data_array(stats_hash)
@@ -154,6 +157,9 @@ module NewRelic
       end
 
       def error_data(unsent_errors)
+        # let the serverless handler handle serialization
+        return NewRelic::Agent.agent.serverless_handler.error_data(unsent_errors) if NewRelic::Agent.agent.serverless?
+
         invoke_remote(:error_data, [@agent_id, unsent_errors],
           :item_count => unsent_errors.size)
       end
@@ -554,6 +560,8 @@ module NewRelic
       # enough to be worth compressing, and handles any errors the
       # server may return
       def invoke_remote(method, payload = [], options = {})
+        return NewRelic::Agent.agent.serverless_handler.store_payload(method, payload) if NewRelic::Agent.agent.serverless?
+
         start_ts = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         request_send_ts, response_check_ts = nil
         data, encoding, size, serialize_finish_ts = marshal_payload(method, payload, options)
@@ -561,8 +569,10 @@ module NewRelic
         response, request_send_ts, response_check_ts = invoke_remote_send_request(method, payload, data, encoding)
         @marshaller.load(decompress_response(response))
       ensure
-        record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
-        record_size_supportability_metrics(method, size, options[:item_count]) if size
+        unless NewRelic::Agent.agent.serverless?
+          record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
+          record_size_supportability_metrics(method, size, options[:item_count]) if size
+        end
       end
 
       def handle_serialization_error(method, e)