datadog 2.7.1 → 2.9.0

data/lib/datadog/core/configuration/settings.rb

@@ -863,6 +863,16 @@ module Datadog
             o.type :float
             o.default 1.0
           end
+
+          # Enable log collection for telemetry. Log collection only works when telemetry is enabled and
+          # logs are enabled.
+          # @default `DD_TELEMETRY_LOG_COLLECTION_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :log_collection_enabled do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_LOG_COLLECTION
+            o.default true
+          end
         end
 
         # Remote configuration

data/lib/datadog/core/configuration.rb

@@ -236,7 +236,7 @@ module Datadog
           rescue ThreadError => e
             logger_without_components.error(
               'Detected deadlock during datadog initialization. ' \
-              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
               "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
             )
             nil
@@ -258,8 +258,16 @@ module Datadog
       def replace_components!(settings, old)
         components = Components.new(settings)
 
+        # Carry over state from existing components to the new ones.
+        # Currently, if we already started the remote component (which
+        # happens after a request goes through installed Rack middleware),
+        # we will start the new remote component as well.
+        old_state = {
+          remote_started: old.remote&.started?,
+        }
+
         old.shutdown!(components)
-        components.startup!(settings)
+        components.startup!(settings, old_state: old_state)
         components
       end
 

data/lib/datadog/{tracing → core}/contrib/rails/utils.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../analytics'
-
 module Datadog
-  module Tracing
+  module Core
     module Contrib
       module Rails
         # common utilities for Rails

data/lib/datadog/core/crashtracking/component.rb

@@ -3,7 +3,6 @@
 require 'libdatadog'
 
 require_relative 'tag_builder'
-require_relative 'agent_base_url'
 require_relative '../utils/only_once'
 require_relative '../utils/at_fork_monkey_patch'
 
@@ -31,8 +30,7 @@ module Datadog
 
         def self.build(settings, agent_settings, logger:)
           tags = TagBuilder.call(settings)
-          agent_base_url = AgentBaseUrl.resolve(agent_settings)
-          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+          agent_base_url = agent_settings.url
 
           ld_library_path = ::Libdatadog.ld_library_path
           logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path

data/lib/datadog/core/remote/client/capabilities.rb

@@ -32,6 +32,12 @@ module Datadog
               register_receivers(Datadog::AppSec::Remote.receivers(@telemetry))
             end
 
+            if settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+              register_capabilities(Datadog::DI::Remote.capabilities)
+              register_products(Datadog::DI::Remote.products)
+              register_receivers(Datadog::DI::Remote.receivers(@telemetry))
+            end
+
             register_capabilities(Datadog::Tracing::Remote.capabilities)
             register_products(Datadog::Tracing::Remote.products)
             register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))

data/lib/datadog/core/remote/client.rb

@@ -24,93 +24,99 @@ module Datadog
           @dispatcher = Dispatcher.new(@capabilities.receivers)
         end
 
-        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
         def sync
           # TODO: Skip sync if no capabilities are registered
           response = transport.send_config(payload)
 
           if response.ok?
-            # when response is completely empty, do nothing as in: leave as is
-            if response.empty?
-              Datadog.logger.debug { 'remote: empty response => NOOP' }
+            process_response(response)
+          elsif response.internal_error?
+            raise TransportError, response.to_s
+          end
+        end
 
-              return
-            end
+        private
 
-            begin
-              paths = response.client_configs.map do |path|
-                Configuration::Path.parse(path)
-              end
+        def process_response(response)
+          # when response is completely empty, do nothing as in: leave as is
+          if response.empty?
+            Datadog.logger.debug { 'remote: empty response => NOOP' }
 
-              targets = Configuration::TargetMap.parse(response.targets)
+            return
+          end
 
-              contents = Configuration::ContentList.parse(response.target_files)
-            rescue Remote::Configuration::Path::ParseError => e
-              raise SyncError, e.message
+          begin
+            paths = response.client_configs.map do |path|
+              Configuration::Path.parse(path)
             end
 
-            # To make sure steep does not complain
-            return unless paths && targets && contents
+            targets = Configuration::TargetMap.parse(response.targets)
+
+            contents = Configuration::ContentList.parse(response.target_files)
+          rescue Remote::Configuration::Path::ParseError => e
+            raise SyncError, e.message
+          end
 
-            # TODO: sometimes it can strangely be so that paths.empty?
-            # TODO: sometimes it can strangely be so that targets.empty?
+          # To make sure steep does not complain
+          return unless paths && targets && contents
 
-            changes = repository.transaction do |current, transaction|
-              # paths to be removed: previously applied paths minus ingress paths
-              (current.paths - paths).each { |p| transaction.delete(p) }
+          # TODO: sometimes it can strangely be so that paths.empty?
+          # TODO: sometimes it can strangely be so that targets.empty?
 
-              # go through each ingress path
-              paths.each do |path|
-                # match target with path
-                target = targets[path]
+          apply_config(paths, targets, contents)
+        end
 
-                # abort entirely if matching target not found
-                raise SyncError, "no target for path '#{path}'" if target.nil?
+        def apply_config(paths, targets, contents)
+          changes = repository.transaction do |current, transaction|
+            # paths to be removed: previously applied paths minus ingress paths
+            (current.paths - paths).each { |p| transaction.delete(p) }
 
-                # new paths are not in previously applied paths
-                new = !current.paths.include?(path)
+            # go through each ingress path
+            paths.each do |path|
+              # match target with path
+              target = targets[path]
 
-                # updated paths are in previously applied paths
-                # but the content hash changed
-                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+              # abort entirely if matching target not found
+              raise SyncError, "no target for path '#{path}'" if target.nil?
 
-                # skip if unchanged
-                same = !new && !changed
+              # new paths are not in previously applied paths
+              new = !current.paths.include?(path)
 
-                next if same
+              # updated paths are in previously applied paths
+              # but the content hash changed
+              changed = current.paths.include?(path) && !current.contents.find_content(path, target)
 
-                # match content with path and target
-                content = contents.find_content(path, target)
+              # skip if unchanged
+              same = !new && !changed
 
-                # abort entirely if matching content not found
-                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+              next if same
 
-                # to be added or updated << config
-                # TODO: metadata (hash, version, etc...)
-                transaction.insert(path, target, content) if new
-                transaction.update(path, target, content) if changed
-              end
+              # match content with path and target
+              content = contents.find_content(path, target)
 
-              # save backend opaque backend state
-              transaction.set(opaque_backend_state: targets.opaque_backend_state)
-              transaction.set(targets_version: targets.version)
+              # abort entirely if matching content not found
+              raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
 
-              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
-              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+              # to be added or updated << config
+              # TODO: metadata (hash, version, etc...)
+              transaction.insert(path, target, content) if new
+              transaction.update(path, target, content) if changed
             end
 
-            if changes.empty?
-              Datadog.logger.debug { 'remote: no changes' }
-            else
-              dispatcher.dispatch(changes, repository)
-            end
-          elsif response.internal_error?
-            raise TransportError, response.to_s
+            # save backend opaque backend state
+            transaction.set(opaque_backend_state: targets.opaque_backend_state)
+            transaction.set(targets_version: targets.version)
+
+            # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+            # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
           end
-        end
-        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
 
-        private
+          if changes.empty?
+            Datadog.logger.debug { 'remote: no changes' }
+          else
+            dispatcher.dispatch(changes, repository)
+          end
+        end
 
         def payload # rubocop:disable Metrics/MethodLength
           state = repository.state

data/lib/datadog/core/telemetry/component.rb

@@ -14,6 +14,7 @@ module Datadog
   module Core
     module Telemetry
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
       class Component
         attr_reader :enabled
 
@@ -52,6 +53,7 @@ module Datadog
             metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
             dependency_collection: settings.telemetry.dependency_collection,
             shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+            log_collection_enabled: settings.telemetry.log_collection_enabled
           )
         end
 
@@ -67,10 +69,11 @@ module Datadog
           http_transport:,
           shutdown_timeout_seconds:,
           enabled: true,
-          metrics_enabled: true
+          metrics_enabled: true,
+          log_collection_enabled: true
         )
           @enabled = enabled
-          @stopped = false
+          @log_collection_enabled = log_collection_enabled
 
           @metrics_manager = MetricsManager.new(
             enabled: enabled && metrics_enabled,
@@ -86,6 +89,9 @@ module Datadog
             dependency_collection: dependency_collection,
             shutdown_timeout: shutdown_timeout_seconds
           )
+
+          @stopped = false
+
           @worker.start
         end
 
@@ -114,7 +120,7 @@ module Datadog
         end
 
         def log!(event)
-          return unless @enabled || forked?
+          return if !@enabled || forked? || !@log_collection_enabled
 
           @worker.enqueue(event)
         end

data/lib/datadog/core/telemetry/event.rb

@@ -29,6 +29,22 @@ module Datadog
           def payload
             {}
           end
+
+          # Override equality to allow for deduplication
+          # The basic implementation is to check if the other object is an instance of the same class.
+          # This works for events that have no attributes.
+          # For events with attributes, you should override this method to compare the attributes.
+          def ==(other)
+            other.is_a?(self.class)
+          end
+
+          # @see #==
+          alias eql? ==
+
+          # @see #==
+          def hash
+            self.class.hash
+          end
         end
 
         # Telemetry class for the 'app-started' event
@@ -263,6 +279,8 @@ module Datadog
 
         # Telemetry class for the 'app-client-configuration-change' event
         class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
           def type
             'app-client-configuration-change'
           end
@@ -301,6 +319,16 @@ module Datadog
 
             res
           end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
         end
 
         # Telemetry class for the 'app-heartbeat' event
@@ -319,6 +347,8 @@ module Datadog
 
         # Telemetry class for the 'generate-metrics' event
         class GenerateMetrics < Base
+          attr_reader :namespace, :metric_series
+
           def type
             'generate-metrics'
           end
@@ -335,24 +365,54 @@ module Datadog
               series: @metric_series.map(&:to_h)
             }
           end
+
+          def ==(other)
+            other.is_a?(GenerateMetrics) && other.namespace == @namespace && other.metric_series == @metric_series
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @namespace, @metric_series].hash
+          end
         end
 
-        # Telemetry class for the 'logs' event
+        # Telemetry class for the 'logs' event.
+        # Logs with the same content are deduplicated at flush time.
         class Log < Base
           LEVELS = {
             error: 'ERROR',
             warn: 'WARN',
           }.freeze
 
+          LEVELS_STRING = LEVELS.values.freeze
+
+          attr_reader :message, :level, :stack_trace, :count
+
           def type
             'logs'
           end
 
-          def initialize(message:, level:, stack_trace: nil)
+          # @param message [String] the log message
+          # @param level [Symbol, String] the log level. Either :error, :warn, 'ERROR', or 'WARN'.
+          # @param stack_trace [String, nil] the stack trace
+          # @param count [Integer] the number of times the log was emitted. Used for deduplication.
+          def initialize(message:, level:, stack_trace: nil, count: 1)
             super()
             @message = message
             @stack_trace = stack_trace
-            @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+
+            if level.is_a?(String) && LEVELS_STRING.include?(level)
+              # String level is used during object copy for deduplication
+              @level = level
+            elsif level.is_a?(Symbol)
+              # Symbol level is used by the regular log emitter user
+              @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+            else
+              raise ArgumentError, "Invalid log level #{level}"
+            end
+
+            @count = count
           end
 
           def payload
@@ -362,10 +422,24 @@ module Datadog
                   message: @message,
                   level: @level,
                   stack_trace: @stack_trace,
+                  count: @count,
                 }.compact
               ]
             }
           end
+
+          # override equality to allow for deduplication
+          def ==(other)
+            other.is_a?(Log) &&
+              other.message == @message &&
+              other.level == @level && other.stack_trace == @stack_trace && other.count == @count
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @message, @level, @stack_trace, @count].hash
+          end
         end
 
         # Telemetry class for the 'distributions' event
@@ -395,6 +469,16 @@ module Datadog
               }
             end
           end
+
+          def ==(other)
+            other.is_a?(MessageBatch) && other.events == @events
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @events].hash
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/ext.rb

@@ -13,6 +13,7 @@ module Datadog
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'
         ENV_AGENTLESS_URL_OVERRIDE = 'DD_TELEMETRY_AGENTLESS_URL'
+        ENV_LOG_COLLECTION = 'DD_TELEMETRY_LOG_COLLECTION_ENABLED'
       end
     end
   end

data/lib/datadog/core/telemetry/logging.rb

@@ -41,7 +41,7 @@ module Datadog
               else
                 'REDACTED'
               end
-            end.join(',')
+            end.join("\n")
           end
         end
 
@@ -49,7 +49,7 @@ module Datadog
           # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
           message = +''
           message << (exception.class.name || exception.class.inspect)
-          message << ':' << description if description
+          message << ': ' << description if description
 
           event = Event::Log.new(
             message: message,

data/lib/datadog/core/telemetry/metric.rb

@@ -41,6 +41,18 @@ module Datadog
             }
           end
 
+          def ==(other)
+            other.is_a?(self.class) &&
+              name == other.name &&
+              values == other.values && tags == other.tags && common == other.common && type == other.type
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, name, values, tags, common, type].hash
+          end
+
           private
 
           def tags_to_array(tags)
@@ -71,6 +83,16 @@ module Datadog
             res[:interval] = interval
             res
           end
+
+          def ==(other)
+            super && interval == other.interval
+          end
+
+          alias eql? ==
+
+          def hash
+            [super, interval].hash
+          end
         end
 
         # Count metric adds up all the submitted values in a time interval. This would be suitable for a

data/lib/datadog/core/telemetry/worker.rb

@@ -97,6 +97,8 @@ module Datadog
           return if events.empty?
           return if !enabled? || !sent_started_event?
 
+          events = deduplicate_logs(events)
+
           Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
           send_event(Event::MessageBatch.new(events))
         end
@@ -167,6 +169,37 @@ module Datadog
           Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
           disable!
         end
+
+        # Deduplicate logs by counting the number of repeated occurrences of the same log
+        # entry and replacing them with a single entry with the calculated `count` value.
+        # Non-log events are unchanged.
+        def deduplicate_logs(events)
+          return events if events.empty?
+
+          all_logs = []
+          other_events = events.reject do |event|
+            if event.is_a?(Event::Log)
+              all_logs << event
+              true
+            else
+              false
+            end
+          end
+
+          return events if all_logs.empty?
+
+          uniq_logs = all_logs.group_by(&:itself).map do |_, logs|
+            log = logs.first
+            if logs.size > 1
+              # New log event with a count of repeated occurrences
+              Event::Log.new(message: log.message, level: log.level, stack_trace: log.stack_trace, count: logs.size)
+            else
+              log
+            end
+          end
+
+          other_events + uniq_logs
+        end
       end
     end
   end