datadog 2.8.0 → 2.9.0

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../instrumentation/gateway'
-require_relative '../../../reactive/operation'
+require_relative '../../../reactive/engine'
 require_relative '../../rack/reactive/request_body'
 require_relative '../reactive/routed'
 require_relative '../../../event'
@@ -23,85 +23,63 @@ module Datadog
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
-                  block = false
-
                   event = nil
-                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
-
-                  AppSec::Reactive::Operation.new('sinatra.request.dispatch') do |op|
-                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result|
-                      if result.status == :match
-                        # TODO: should this hash be an Event instance instead?
-                        event = {
-                          waf_result: result,
-                          trace: scope.trace,
-                          span: scope.service_entry_span,
-                          request: gateway_request,
-                          actions: result.actions
-                        }
-
-                        # We want to keep the trace in case of security event
-                        scope.trace.keep! if scope.trace
-                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                        scope.processor_context.events << event
-                      end
+                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  engine = AppSec::Reactive::Engine.new
+
+                  Rack::Reactive::RequestBody.subscribe(engine, context) do |result|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: context.trace,
+                        span: context.span,
+                        request: gateway_request,
+                        actions: result.actions
+                      }
+
+                      # We want to keep the trace in case of security event
+                      context.trace.keep! if context.trace
+                      Datadog::AppSec::Event.tag_and_keep!(context, result)
+                      context.waf_runner.events << event
                     end
-
-                    block = Rack::Reactive::RequestBody.publish(op, gateway_request)
                   end
 
+                  block = Rack::Reactive::RequestBody.publish(engine, gateway_request)
                   next [nil, [[:block, event]]] if block
 
-                  ret, res = stack.call(gateway_request.request)
-
-                  if event
-                    res ||= []
-                    res << [:monitor, event]
-                  end
-
-                  [ret, res]
+                  stack.call(gateway_request.request)
                 end
               end
 
               def watch_request_routed(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
-                  block = false
-
                   event = nil
-                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
-
-                  AppSec::Reactive::Operation.new('sinatra.request.routed') do |op|
-                    Sinatra::Reactive::Routed.subscribe(op, scope.processor_context) do |result|
-                      if result.status == :match
-                        # TODO: should this hash be an Event instance instead?
-                        event = {
-                          waf_result: result,
-                          trace: scope.trace,
-                          span: scope.service_entry_span,
-                          request: gateway_request,
-                          actions: result.actions
-                        }
-
-                        # We want to keep the trace in case of security event
-                        scope.trace.keep! if scope.trace
-                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                        scope.processor_context.events << event
-                      end
+                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  engine = AppSec::Reactive::Engine.new
+
+                  Sinatra::Reactive::Routed.subscribe(engine, context) do |result|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: context.trace,
+                        span: context.span,
+                        request: gateway_request,
+                        actions: result.actions
+                      }
+
+                      # We want to keep the trace in case of security event
+                      context.trace.keep! if context.trace
+                      Datadog::AppSec::Event.tag_and_keep!(context, result)
+                      context.waf_runner.events << event
                     end
-
-                    block = Sinatra::Reactive::Routed.publish(op, [gateway_request, gateway_route_params])
                   end
 
+                  block = Sinatra::Reactive::Routed.publish(engine, [gateway_request, gateway_route_params])
                   next [nil, [[:block, event]]] if block
 
-                  ret, res = stack.call(gateway_request.request)
-
-                  if event
-                    res ||= []
-                    res << [:monitor, event]
-                  end
-
-                  [ret, res]
+                  stack.call(gateway_request.request)
                 end
               end
             end

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -54,7 +54,7 @@ module Datadog
           def dispatch!
             env = @request.env
 
-            context = env[Datadog::AppSec::Ext::SCOPE_KEY]
+            context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
 
             return super unless context
 
@@ -86,7 +86,7 @@ module Datadog
           def process_route(*)
             env = @request.env
 
-            context = env[Datadog::AppSec::Ext::SCOPE_KEY]
+            context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
 
             return super unless context
 

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -12,18 +12,18 @@ module Datadog
             ].freeze
             private_constant :ADDRESSES
 
-            def self.publish(op, data)
+            def self.publish(engine, data)
               _request, route_params = data
 
               catch(:block) do
-                op.publish('sinatra.request.route_params', route_params.params)
+                engine.publish('sinatra.request.route_params', route_params.params)
 
                 nil
               end
             end
 
-            def self.subscribe(op, waf_context)
-              op.subscribe(*ADDRESSES) do |*values|
+            def self.subscribe(engine, context)
+              engine.subscribe(*ADDRESSES) do |*values|
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
                 path_params = values[0]
 
@@ -32,7 +32,7 @@ module Datadog
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(persistent_data, {}, waf_timeout)
+                result = context.run_waf(persistent_data, {}, waf_timeout)
 
                 next if result.status != :match
 

data/lib/datadog/appsec/event.rb

@@ -137,16 +137,16 @@ module Datadog
         end
         # rubocop:enable Metrics/MethodLength
 
-        def tag_and_keep!(scope, waf_result)
+        def tag_and_keep!(context, waf_result)
           # We want to keep the trace in case of security event
-          scope.trace.keep! if scope.trace
+          context.trace.keep! if context.trace
 
-          if scope.service_entry_span
-            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
-            scope.service_entry_span.set_tag('appsec.event', 'true')
+          if context.span
+            context.span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
+            context.span.set_tag('appsec.event', 'true')
           end
 
-          add_distributed_tags(scope.trace)
+          add_distributed_tags(context.trace)
         end
 
         private

data/lib/datadog/appsec/ext.rb

@@ -3,8 +3,10 @@
 module Datadog
   module AppSec
     module Ext
+      RASP_SQLI = :sql_injection
       INTERRUPT = :datadog_appsec_interrupt
-      SCOPE_KEY = 'datadog.appsec.scope'
+      CONTEXT_KEY = 'datadog.appsec.context'
+      ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
       TAG_APM_ENABLED = '_dd.apm.enabled'

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../instrumentation/gateway'
-require_relative '../../reactive/operation'
+require_relative '../../reactive/engine'
 require_relative '../reactive/set_user'
 
 module Datadog
@@ -19,42 +19,32 @@ module Datadog
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                block = false
                 event = nil
-                scope = Datadog::AppSec.active_scope
-
-                AppSec::Reactive::Operation.new('identity.set_user') do |op|
-                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: scope.trace,
-                        span: scope.service_entry_span,
-                        user: user,
-                        actions: result.actions
-                      }
-
-                      # We want to keep the trace in case of security event
-                      scope.trace.keep! if scope.trace
-                      Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                      scope.processor_context.events << event
-                    end
+                context = Datadog::AppSec.active_context
+                engine = AppSec::Reactive::Engine.new
+
+                Monitor::Reactive::SetUser.subscribe(engine, context) do |result|
+                  if result.status == :match
+                    # TODO: should this hash be an Event instance instead?
+                    event = {
+                      waf_result: result,
+                      trace: context.trace,
+                      span: context.span,
+                      user: user,
+                      actions: result.actions
+                    }
+
+                    # We want to keep the trace in case of security event
+                    context.trace.keep! if context.trace
+                    Datadog::AppSec::Event.tag_and_keep!(context, result)
+                    context.waf_runner.events << event
                   end
-
-                  block = Monitor::Reactive::SetUser.publish(op, user)
                 end
 
-                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
-
-                ret, res = stack.call(user)
-
-                if event
-                  res ||= []
-                  res << [:monitor, event]
-                end
+                block = Monitor::Reactive::SetUser.publish(engine, user)
+                throw(Datadog::AppSec::Ext::INTERRUPT, event[:actions]) if block
 
-                [ret, res]
+                stack.call(user)
               end
             end
           end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -11,16 +11,16 @@ module Datadog
           ].freeze
           private_constant :ADDRESSES
 
-          def self.publish(op, user)
+          def self.publish(engine, user)
             catch(:block) do
-              op.publish('usr.id', user.id)
+              engine.publish('usr.id', user.id)
 
               nil
             end
           end
 
-          def self.subscribe(op, waf_context)
-            op.subscribe(*ADDRESSES) do |*values|
+          def self.subscribe(engine, context)
+            engine.subscribe(*ADDRESSES) do |*values|
               Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
 
               user_id = values[0]
@@ -30,7 +30,7 @@ module Datadog
               }
 
               waf_timeout = Datadog.configuration.appsec.waf_timeout
-              result = waf_context.run(persistent_data, {}, waf_timeout)
+              result = context.run_waf(persistent_data, {}, waf_timeout)
 
               next if result.status != :match
 

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -74,9 +74,6 @@ module Datadog
             when Hash
               pass = ip_passlist[:pass]
               monitor = ip_passlist[:monitor]
-            else
-              pass = []
-              monitor = []
             end
 
             exclusions = []

data/lib/datadog/appsec.rb

@@ -2,7 +2,7 @@
 
 require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
-require_relative 'appsec/scope'
+require_relative 'appsec/context'
 require_relative 'appsec/ext'
 require_relative 'appsec/utils'
 
@@ -14,8 +14,8 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
-      def active_scope
-        Datadog::AppSec::Scope.active_scope
+      def active_context
+        Datadog::AppSec::Context.active
       end
 
       def processor

data/lib/datadog/auto_instrument.rb

@@ -6,6 +6,9 @@
 require_relative '../datadog'
 require_relative 'tracing/contrib/auto_instrument'
 
+# DI is not loaded on Ruby 2.5 and JRuby
+Datadog::DI::Contrib.load_now_or_later if defined?(Datadog::DI::Contrib)
+
 Datadog::Profiling.start_if_enabled
 
 module Datadog

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -19,21 +19,49 @@ module Datadog
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
       class AgentSettingsResolver
-        AgentSettings = Struct.new(
-          :adapter,
-          :ssl,
-          :hostname,
-          :port,
-          :uds_path,
-          :timeout_seconds,
-          keyword_init: true
-        ) do
-          def initialize(*)
-            super
+        # Immutable container for the resulting settings
+        class AgentSettings
+          attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
+
+          def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
+            @adapter = adapter
+            @ssl = ssl
+            @hostname = hostname
+            @port = port
+            @uds_path = uds_path
+            @timeout_seconds = timeout_seconds
             freeze
           end
+
+          def url
+            case adapter
+            when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              hostname = self.hostname
+              hostname = "[#{hostname}]" if hostname =~ IPV6_REGEXP
+              "#{ssl ? 'https' : 'http'}://#{hostname}:#{port}/"
+            when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+              "unix://#{uds_path}"
+            else
+              raise ArgumentError, "Unexpected adapter: #{adapter}"
+            end
+          end
+
+          def ==(other)
+            self.class == other.class &&
+              adapter == other.adapter &&
+              ssl == other.ssl &&
+              hostname == other.hostname &&
+              port == other.port &&
+              uds_path == other.uds_path &&
+              timeout_seconds == other.timeout_seconds
+          end
         end
 
+        # IPv6 regular expression from
+        # https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
+        # Does not match IPv4 addresses.
+        IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
+
         def self.call(settings, logger: Datadog.logger)
           new(settings, logger: logger).send(:call)
         end

data/lib/datadog/core/configuration/components.rb

@@ -105,14 +105,16 @@ module Datadog
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer
+            optional_tracer: @tracer,
+            logger: @logger,
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
           @health_metrics = self.class.build_health_metrics(settings)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
-          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, telemetry: telemetry)
+          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
 
           self.class.configure_tracing(settings)
         end

data/lib/datadog/core/configuration.rb

@@ -236,7 +236,7 @@ module Datadog
           rescue ThreadError => e
             logger_without_components.error(
               'Detected deadlock during datadog initialization. ' \
-              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
               "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
             )
             nil

data/lib/datadog/{tracing → core}/contrib/rails/utils.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../analytics'
-
 module Datadog
-  module Tracing
+  module Core
     module Contrib
       module Rails
         # common utilities for Rails

data/lib/datadog/core/crashtracking/component.rb

@@ -3,7 +3,6 @@
 require 'libdatadog'
 
 require_relative 'tag_builder'
-require_relative 'agent_base_url'
 require_relative '../utils/only_once'
 require_relative '../utils/at_fork_monkey_patch'
 
@@ -31,8 +30,7 @@ module Datadog
 
         def self.build(settings, agent_settings, logger:)
           tags = TagBuilder.call(settings)
-          agent_base_url = AgentBaseUrl.resolve(agent_settings)
-          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+          agent_base_url = agent_settings.url
 
           ld_library_path = ::Libdatadog.ld_library_path
           logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path

data/lib/datadog/core/telemetry/event.rb

@@ -29,6 +29,22 @@ module Datadog
           def payload
             {}
           end
+
+          # Override equality to allow for deduplication
+          # The basic implementation is to check if the other object is an instance of the same class.
+          # This works for events that have no attributes.
+          # For events with attributes, you should override this method to compare the attributes.
+          def ==(other)
+            other.is_a?(self.class)
+          end
+
+          # @see #==
+          alias eql? ==
+
+          # @see #==
+          def hash
+            self.class.hash
+          end
         end
 
         # Telemetry class for the 'app-started' event
@@ -263,6 +279,8 @@ module Datadog
 
         # Telemetry class for the 'app-client-configuration-change' event
         class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
           def type
             'app-client-configuration-change'
           end
@@ -301,6 +319,16 @@ module Datadog
 
             res
           end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
         end
 
         # Telemetry class for the 'app-heartbeat' event
@@ -319,6 +347,8 @@ module Datadog
 
         # Telemetry class for the 'generate-metrics' event
         class GenerateMetrics < Base
+          attr_reader :namespace, :metric_series
+
           def type
             'generate-metrics'
           end
@@ -335,24 +365,54 @@ module Datadog
               series: @metric_series.map(&:to_h)
             }
           end
+
+          def ==(other)
+            other.is_a?(GenerateMetrics) && other.namespace == @namespace && other.metric_series == @metric_series
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @namespace, @metric_series].hash
+          end
         end
 
-        # Telemetry class for the 'logs' event
+        # Telemetry class for the 'logs' event.
+        # Logs with the same content are deduplicated at flush time.
         class Log < Base
           LEVELS = {
             error: 'ERROR',
             warn: 'WARN',
           }.freeze
 
+          LEVELS_STRING = LEVELS.values.freeze
+
+          attr_reader :message, :level, :stack_trace, :count
+
           def type
             'logs'
           end
 
-          def initialize(message:, level:, stack_trace: nil)
+          # @param message [String] the log message
+          # @param level [Symbol, String] the log level. Either :error, :warn, 'ERROR', or 'WARN'.
+          # @param stack_trace [String, nil] the stack trace
+          # @param count [Integer] the number of times the log was emitted. Used for deduplication.
+          def initialize(message:, level:, stack_trace: nil, count: 1)
             super()
             @message = message
             @stack_trace = stack_trace
-            @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+
+            if level.is_a?(String) && LEVELS_STRING.include?(level)
+              # String level is used during object copy for deduplication
+              @level = level
+            elsif level.is_a?(Symbol)
+              # Symbol level is used by the regular log emitter user
+              @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+            else
+              raise ArgumentError, "Invalid log level #{level}"
+            end
+
+            @count = count
           end
 
           def payload
@@ -362,10 +422,24 @@ module Datadog
                   message: @message,
                   level: @level,
                   stack_trace: @stack_trace,
+                  count: @count,
                 }.compact
               ]
             }
           end
+
+          # override equality to allow for deduplication
+          def ==(other)
+            other.is_a?(Log) &&
+              other.message == @message &&
+              other.level == @level && other.stack_trace == @stack_trace && other.count == @count
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @message, @level, @stack_trace, @count].hash
+          end
         end
 
         # Telemetry class for the 'distributions' event
@@ -395,6 +469,16 @@ module Datadog
               }
             end
           end
+
+          def ==(other)
+            other.is_a?(MessageBatch) && other.events == @events
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @events].hash
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/logging.rb

@@ -41,7 +41,7 @@ module Datadog
               else
                 'REDACTED'
               end
-            end.join(',')
+            end.join("\n")
           end
         end
 
@@ -49,7 +49,7 @@ module Datadog
           # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
           message = +''
           message << (exception.class.name || exception.class.inspect)
-          message << ':' << description if description
+          message << ': ' << description if description
 
           event = Event::Log.new(
             message: message,