datadog 2.8.0 → 2.10.0

data/lib/datadog/appsec/response.rb

@@ -19,100 +19,38 @@ module Datadog
         [status, headers, body]
       end
 
-      def to_sinatra_response
-        ::Sinatra::Response.new(body, status, headers)
-      end
-
-      def to_action_dispatch_response
-        ::ActionDispatch::Response.new(status, headers, body)
-      end
-
       class << self
-        def negotiate(env, actions)
-          # @type var configured_response: Response?
-          configured_response = nil
-          actions.each do |type, parameters|
-            # Need to use next to make steep happy :(
-            # I rather use break to stop the execution
-            next if configured_response
-
-            configured_response = case type
-                                  when 'block_request'
-                                    block_response(env, parameters)
-                                  when 'redirect_request'
-                                    redirect_response(env, parameters)
-                                  end
-          end
-
-          configured_response || default_response(env)
-        end
-
-        def graphql_response(gateway_multiplex)
-          multiplex_return = []
-          gateway_multiplex.queries.each do |query|
-            # This method is only called in places where GraphQL-Ruby is already required
-            query_result = ::GraphQL::Query::Result.new(
-              query: query,
-              values: JSON.parse(content('application/json'))
-            )
-            multiplex_return << query_result
-          end
+        def from_interrupt_params(interrupt_params, http_accept_header)
+          return redirect_response(interrupt_params) if interrupt_params['location']
 
-          multiplex_return
+          block_response(interrupt_params, http_accept_header)
         end
 
         private
 
-        def default_response(env)
-          content_type = content_type(env)
-
-          body = []
-          body << content(content_type)
+        def block_response(interrupt_params, http_accept_header)
+          content_type = case interrupt_params['type']
+                         when nil, 'auto' then content_type(http_accept_header)
+                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+                         end
 
           Response.new(
-            status: 403,
+            status: interrupt_params['status_code']&.to_i || 403,
             headers: { 'Content-Type' => content_type },
-            body: body,
+            body: [content(content_type)],
           )
         end
 
-        def block_response(env, options)
-          content_type = if options['type'] == 'auto'
-                           content_type(env)
-                         else
-                           FORMAT_TO_CONTENT_TYPE[options['type']]
-                         end
-
-          body = []
-          body << content(content_type)
+        def redirect_response(interrupt_params)
+          status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: options['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
-            body: body,
+            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
+            headers: { 'Location' => interrupt_params.fetch('location') },
+            body: [],
           )
         end
 
-        def redirect_response(env, options)
-          if options['location'] && !options['location'].empty?
-            content_type = content_type(env)
-
-            headers = {
-              'Content-Type' => content_type,
-              'Location' => options['location']
-            }
-
-            status_code = options['status_code'].to_i
-            Response.new(
-              status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-              headers: headers,
-              body: [],
-            )
-          else
-            default_response(env)
-          end
-        end
-
         CONTENT_TYPE_TO_FORMAT = {
           'application/json' => :json,
           'text/html' => :html,
@@ -126,10 +64,10 @@ module Datadog
 
         DEFAULT_CONTENT_TYPE = 'application/json'
 
-        def content_type(env)
-          return DEFAULT_CONTENT_TYPE unless env.key?('HTTP_ACCEPT')
+        def content_type(http_accept_header)
+          return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = env['HTTP_ACCEPT'].split(',').map(&:strip)
+          accept_types = http_accept_header.split(',').map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+            @events = events
+            @actions = actions
+            @derivatives = derivatives
+
+            @timeout = timeout
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            !!@timeout
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:)
+            @events = []
+            @actions = @derivatives = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            false
+          end
+
+          def match?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A class that check input via security engine (WAF) and respond with result.
+      class Runner
+        SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        def initialize(handle, telemetry:)
+          @mutex = Mutex.new
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          report_execution(result)
+
+          unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+          end
+
+          klass = result.status == :match ? Result::Match : Result::Ok
+          klass.new(
+            events: result.events,
+            actions: result.actions,
+            derivatives: result.derivatives,
+            timeout: result.timeout,
+            duration_ns: result.total_runtime,
+            duration_ext_ns: (stop_ns - start_ns)
+          )
+        ensure
+          @mutex.unlock
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A namespace for secutiry library we use to detect and prevent threats.
+    module SecurityEngine
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -2,7 +2,7 @@
 
 require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
-require_relative 'appsec/scope'
+require_relative 'appsec/context'
 require_relative 'appsec/ext'
 require_relative 'appsec/utils'
 
@@ -14,19 +14,24 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
-      def active_scope
-        Datadog::AppSec::Scope.active_scope
+      def rasp_enabled?
+        Datadog.configuration.appsec.rasp_enabled
       end
 
-      def processor
-        appsec_component = components.appsec
+      def active_context
+        Datadog::AppSec::Context.active
+      end
+
+      def telemetry
+        components.appsec&.telemetry
+      end
 
-        appsec_component.processor if appsec_component
+      def processor
+        components.appsec&.processor
       end
 
       def reconfigure(ruleset:, telemetry:)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
@@ -34,12 +39,16 @@ module Datadog
 
       def reconfigure_lock(&block)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure_lock(&block)
       end
 
+      def api_security_enabled?
+        Datadog.configuration.appsec.api_security.enabled &&
+          Datadog.configuration.appsec.api_security.sample_rate.sample?
+      end
+
       private
 
       def components

data/lib/datadog/auto_instrument.rb

@@ -6,6 +6,9 @@
 require_relative '../datadog'
 require_relative 'tracing/contrib/auto_instrument'
 
+# DI is not loaded on Ruby 2.5 and JRuby
+Datadog::DI::Contrib.load_now_or_later if defined?(Datadog::DI::Contrib)
+
 Datadog::Profiling.start_if_enabled
 
 module Datadog

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -19,21 +19,49 @@ module Datadog
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
       class AgentSettingsResolver
-        AgentSettings = Struct.new(
-          :adapter,
-          :ssl,
-          :hostname,
-          :port,
-          :uds_path,
-          :timeout_seconds,
-          keyword_init: true
-        ) do
-          def initialize(*)
-            super
+        # Immutable container for the resulting settings
+        class AgentSettings
+          attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
+
+          def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
+            @adapter = adapter
+            @ssl = ssl
+            @hostname = hostname
+            @port = port
+            @uds_path = uds_path
+            @timeout_seconds = timeout_seconds
             freeze
           end
+
+          def url
+            case adapter
+            when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              hostname = self.hostname
+              hostname = "[#{hostname}]" if hostname =~ IPV6_REGEXP
+              "#{ssl ? 'https' : 'http'}://#{hostname}:#{port}/"
+            when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+              "unix://#{uds_path}"
+            else
+              raise ArgumentError, "Unexpected adapter: #{adapter}"
+            end
+          end
+
+          def ==(other)
+            self.class == other.class &&
+              adapter == other.adapter &&
+              ssl == other.ssl &&
+              hostname == other.hostname &&
+              port == other.port &&
+              uds_path == other.uds_path &&
+              timeout_seconds == other.timeout_seconds
+          end
         end
 
+        # IPv6 regular expression from
+        # https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
+        # Does not match IPv4 addresses.
+        IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
+
         def self.call(settings, logger: Datadog.logger)
           new(settings, logger: logger).send(:call)
         end

data/lib/datadog/core/configuration/components.rb

@@ -105,14 +105,16 @@ module Datadog
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer
+            optional_tracer: @tracer,
+            logger: @logger,
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
           @health_metrics = self.class.build_health_metrics(settings)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
-          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, telemetry: telemetry)
+          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
 
           self.class.configure_tracing(settings)
         end

data/lib/datadog/core/configuration.rb

@@ -236,7 +236,7 @@ module Datadog
           rescue ThreadError => e
             logger_without_components.error(
               'Detected deadlock during datadog initialization. ' \
-              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
               "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
             )
             nil

data/lib/datadog/{tracing → core}/contrib/rails/utils.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../analytics'
-
 module Datadog
-  module Tracing
+  module Core
     module Contrib
       module Rails
         # common utilities for Rails

data/lib/datadog/core/crashtracking/component.rb

@@ -3,7 +3,6 @@
 require 'libdatadog'
 
 require_relative 'tag_builder'
-require_relative 'agent_base_url'
 require_relative '../utils/only_once'
 require_relative '../utils/at_fork_monkey_patch'
 
@@ -31,8 +30,7 @@ module Datadog
 
         def self.build(settings, agent_settings, logger:)
           tags = TagBuilder.call(settings)
-          agent_base_url = AgentBaseUrl.resolve(agent_settings)
-          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+          agent_base_url = agent_settings.url
 
           ld_library_path = ::Libdatadog.ld_library_path
           logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path

data/lib/datadog/core/telemetry/event.rb

@@ -29,6 +29,22 @@ module Datadog
           def payload
             {}
           end
+
+          # Override equality to allow for deduplication
+          # The basic implementation is to check if the other object is an instance of the same class.
+          # This works for events that have no attributes.
+          # For events with attributes, you should override this method to compare the attributes.
+          def ==(other)
+            other.is_a?(self.class)
+          end
+
+          # @see #==
+          alias eql? ==
+
+          # @see #==
+          def hash
+            self.class.hash
+          end
         end
 
         # Telemetry class for the 'app-started' event
@@ -263,6 +279,8 @@ module Datadog
 
         # Telemetry class for the 'app-client-configuration-change' event
         class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
           def type
             'app-client-configuration-change'
           end
@@ -301,6 +319,16 @@ module Datadog
 
             res
           end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
         end
 
         # Telemetry class for the 'app-heartbeat' event
@@ -319,6 +347,8 @@ module Datadog
 
         # Telemetry class for the 'generate-metrics' event
         class GenerateMetrics < Base
+          attr_reader :namespace, :metric_series
+
           def type
             'generate-metrics'
           end
@@ -335,24 +365,54 @@ module Datadog
               series: @metric_series.map(&:to_h)
             }
           end
+
+          def ==(other)
+            other.is_a?(GenerateMetrics) && other.namespace == @namespace && other.metric_series == @metric_series
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @namespace, @metric_series].hash
+          end
         end
 
-        # Telemetry class for the 'logs' event
+        # Telemetry class for the 'logs' event.
+        # Logs with the same content are deduplicated at flush time.
         class Log < Base
           LEVELS = {
             error: 'ERROR',
             warn: 'WARN',
           }.freeze
 
+          LEVELS_STRING = LEVELS.values.freeze
+
+          attr_reader :message, :level, :stack_trace, :count
+
           def type
             'logs'
           end
 
-          def initialize(message:, level:, stack_trace: nil)
+          # @param message [String] the log message
+          # @param level [Symbol, String] the log level. Either :error, :warn, 'ERROR', or 'WARN'.
+          # @param stack_trace [String, nil] the stack trace
+          # @param count [Integer] the number of times the log was emitted. Used for deduplication.
+          def initialize(message:, level:, stack_trace: nil, count: 1)
             super()
             @message = message
             @stack_trace = stack_trace
-            @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+
+            if level.is_a?(String) && LEVELS_STRING.include?(level)
+              # String level is used during object copy for deduplication
+              @level = level
+            elsif level.is_a?(Symbol)
+              # Symbol level is used by the regular log emitter user
+              @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+            else
+              raise ArgumentError, "Invalid log level #{level}"
+            end
+
+            @count = count
           end
 
           def payload
@@ -362,10 +422,24 @@ module Datadog
                   message: @message,
                   level: @level,
                   stack_trace: @stack_trace,
+                  count: @count,
                 }.compact
               ]
             }
           end
+
+          # override equality to allow for deduplication
+          def ==(other)
+            other.is_a?(Log) &&
+              other.message == @message &&
+              other.level == @level && other.stack_trace == @stack_trace && other.count == @count
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @message, @level, @stack_trace, @count].hash
+          end
         end
 
         # Telemetry class for the 'distributions' event
@@ -395,6 +469,16 @@ module Datadog
               }
             end
           end
+
+          def ==(other)
+            other.is_a?(MessageBatch) && other.events == @events
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, @events].hash
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/logging.rb

@@ -41,7 +41,7 @@ module Datadog
               else
                 'REDACTED'
               end
-            end.join(',')
+            end.join("\n")
           end
         end
 
@@ -49,7 +49,7 @@ module Datadog
           # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
           message = +''
           message << (exception.class.name || exception.class.inspect)
-          message << ':' << description if description
+          message << ': ' << description if description
 
           event = Event::Log.new(
             message: message,