datadog 2.9.0 → 2.11.0

data/lib/datadog/appsec/metrics.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This namespace contains classes related to metrics collection and exportation.
+    module Metrics
+    end
+  end
+end
+
+require_relative 'metrics/collector'
+require_relative 'metrics/exporter'
+require_relative 'metrics/telemetry'

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../../instrumentation/gateway'
-require_relative '../../reactive/engine'
-require_relative '../reactive/set_user'
 
 module Datadog
   module AppSec
@@ -19,30 +17,27 @@ module Datadog
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                event = nil
                 context = Datadog::AppSec.active_context
-                engine = AppSec::Reactive::Engine.new
-
-                Monitor::Reactive::SetUser.subscribe(engine, context) do |result|
-                  if result.status == :match
-                    # TODO: should this hash be an Event instance instead?
-                    event = {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      user: user,
-                      actions: result.actions
-                    }
-
-                    # We want to keep the trace in case of security event
-                    context.trace.keep! if context.trace
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-                    context.waf_runner.events << event
-                  end
-                end
 
-                block = Monitor::Reactive::SetUser.publish(engine, user)
-                throw(Datadog::AppSec::Ext::INTERRUPT, event[:actions]) if block
+                persistent_data = {
+                  'usr.id' => user.id
+                }
+
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match?
+                  Datadog::AppSec::Event.tag_and_keep!(context, result)
+
+                  context.events << {
+                    waf_result: result,
+                    trace: context.trace,
+                    span: context.span,
+                    user: user,
+                    actions: result.actions
+                  }
+
+                  Datadog::AppSec::ActionsHandler.handle(result.actions)
+                end
 
                 stack.call(user)
               end

data/lib/datadog/appsec/processor.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'processor/context'
+require_relative 'security_engine/runner'
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
+    # NOTE: This class will be moved under AppSec::SecurityEngine namespace
     class Processor
       attr_reader :diagnostics, :addresses
 
@@ -29,8 +30,8 @@ module Datadog
         @handle.finalize
       end
 
-      def new_context
-        Context.new(@handle, telemetry: @telemetry)
+      def new_runner
+        SecurityEngine::Runner.new(@handle, telemetry: @telemetry)
       end
 
       private

data/lib/datadog/appsec/remote.rb

@@ -104,6 +104,10 @@ module Datadog
             )
 
             Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
+
+            repository.contents.each do |content|
+              content.applied if ASM_PRODUCTS.include?(content.path.product)
+            end
           end
 
           [receiver]

data/lib/datadog/appsec/response.rb

@@ -19,100 +19,38 @@ module Datadog
         [status, headers, body]
       end
 
-      def to_sinatra_response
-        ::Sinatra::Response.new(body, status, headers)
-      end
-
-      def to_action_dispatch_response
-        ::ActionDispatch::Response.new(status, headers, body)
-      end
-
       class << self
-        def negotiate(env, actions)
-          # @type var configured_response: Response?
-          configured_response = nil
-          actions.each do |type, parameters|
-            # Need to use next to make steep happy :(
-            # I rather use break to stop the execution
-            next if configured_response
-
-            configured_response = case type
-                                  when 'block_request'
-                                    block_response(env, parameters)
-                                  when 'redirect_request'
-                                    redirect_response(env, parameters)
-                                  end
-          end
-
-          configured_response || default_response(env)
-        end
-
-        def graphql_response(gateway_multiplex)
-          multiplex_return = []
-          gateway_multiplex.queries.each do |query|
-            # This method is only called in places where GraphQL-Ruby is already required
-            query_result = ::GraphQL::Query::Result.new(
-              query: query,
-              values: JSON.parse(content('application/json'))
-            )
-            multiplex_return << query_result
-          end
+        def from_interrupt_params(interrupt_params, http_accept_header)
+          return redirect_response(interrupt_params) if interrupt_params['location']
 
-          multiplex_return
+          block_response(interrupt_params, http_accept_header)
         end
 
         private
 
-        def default_response(env)
-          content_type = content_type(env)
-
-          body = []
-          body << content(content_type)
+        def block_response(interrupt_params, http_accept_header)
+          content_type = case interrupt_params['type']
+                         when nil, 'auto' then content_type(http_accept_header)
+                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+                         end
 
           Response.new(
-            status: 403,
+            status: interrupt_params['status_code']&.to_i || 403,
             headers: { 'Content-Type' => content_type },
-            body: body,
+            body: [content(content_type)],
           )
         end
 
-        def block_response(env, options)
-          content_type = if options['type'] == 'auto'
-                           content_type(env)
-                         else
-                           FORMAT_TO_CONTENT_TYPE[options['type']]
-                         end
-
-          body = []
-          body << content(content_type)
+        def redirect_response(interrupt_params)
+          status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: options['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
-            body: body,
+            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
+            headers: { 'Location' => interrupt_params.fetch('location') },
+            body: [],
           )
         end
 
-        def redirect_response(env, options)
-          if options['location'] && !options['location'].empty?
-            content_type = content_type(env)
-
-            headers = {
-              'Content-Type' => content_type,
-              'Location' => options['location']
-            }
-
-            status_code = options['status_code'].to_i
-            Response.new(
-              status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-              headers: headers,
-              body: [],
-            )
-          else
-            default_response(env)
-          end
-        end
-
         CONTENT_TYPE_TO_FORMAT = {
           'application/json' => :json,
           'text/html' => :html,
@@ -126,10 +64,10 @@ module Datadog
 
         DEFAULT_CONTENT_TYPE = 'application/json'
 
-        def content_type(env)
-          return DEFAULT_CONTENT_TYPE unless env.key?('HTTP_ACCEPT')
+        def content_type(http_accept_header)
+          return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = env['HTTP_ACCEPT'].split(',').map(&:strip)
+          accept_types = http_accept_header.split(',').map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+            @events = events
+            @actions = actions
+            @derivatives = derivatives
+
+            @timeout = timeout
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            !!@timeout
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:)
+            @events = []
+            @actions = @derivatives = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            false
+          end
+
+          def match?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A class that check input via security engine (WAF) and respond with result.
+      class Runner
+        SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        def initialize(handle, telemetry:)
+          @mutex = Mutex.new
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          report_execution(result)
+
+          unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+          end
+
+          klass = result.status == :match ? Result::Match : Result::Ok
+          klass.new(
+            events: result.events,
+            actions: result.actions,
+            derivatives: result.derivatives,
+            timeout: result.timeout,
+            duration_ns: result.total_runtime,
+            duration_ext_ns: (stop_ns - start_ns)
+          )
+        ensure
+          @mutex.unlock
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A namespace for secutiry library we use to detect and prevent threats.
+    module SecurityEngine
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -14,19 +14,24 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
+      def rasp_enabled?
+        Datadog.configuration.appsec.rasp_enabled
+      end
+
       def active_context
         Datadog::AppSec::Context.active
       end
 
-      def processor
-        appsec_component = components.appsec
+      def telemetry
+        components.appsec&.telemetry
+      end
 
-        appsec_component.processor if appsec_component
+      def processor
+        components.appsec&.processor
       end
 
       def reconfigure(ruleset:, telemetry:)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
@@ -34,12 +39,16 @@ module Datadog
 
       def reconfigure_lock(&block)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure_lock(&block)
       end
 
+      def api_security_enabled?
+        Datadog.configuration.appsec.api_security.enabled &&
+          Datadog.configuration.appsec.api_security.sample_rate.sample?
+      end
+
       private
 
       def components
@@ -59,5 +68,7 @@ require_relative 'appsec/contrib/rails/integration'
 require_relative 'appsec/contrib/active_record/integration'
 require_relative 'appsec/contrib/devise/integration'
 require_relative 'appsec/contrib/graphql/integration'
+require_relative 'appsec/contrib/faraday/integration'
+require_relative 'appsec/contrib/excon/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/core/configuration/components.rb

@@ -16,6 +16,8 @@ require_relative '../../appsec/component'
 require_relative '../../di/component'
 require_relative '../crashtracking/component'
 
+require_relative '../environment/agent_info'
+
 module Datadog
   module Core
     module Configuration
@@ -85,7 +87,8 @@ module Datadog
           :tracer,
           :crashtracker,
           :dynamic_instrumentation,
-          :appsec
+          :appsec,
+          :agent_info
 
         def initialize(settings)
           @logger = self.class.build_logger(settings)
@@ -96,6 +99,9 @@ module Datadog
           # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
+          # Exposes agent capability information for detection by any components
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings)
+
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
           @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)

data/lib/datadog/core/configuration/ext.rb

@@ -37,7 +37,7 @@ module Datadog
           module UnixSocket
             ADAPTER = :unix
             DEFAULT_PATH = '/var/run/datadog/apm.socket'
-            DEFAULT_TIMEOUT_SECONDS = 1
+            DEFAULT_TIMEOUT_SECONDS = 30
           end
         end
       end

data/lib/datadog/core/configuration/option_definition.rb

@@ -79,6 +79,8 @@ module Datadog
             @deprecated_env = value
           end
 
+          # Invoked when the option is first read, and {#env} is defined.
+          # The block provided is only invoked if the environment variable is present (not-nil).
           def env_parser(&block)
             @env_parser = block
           end

data/lib/datadog/core/configuration/settings.rb

@@ -461,15 +461,31 @@ module Datadog
               end
             end
 
-            # Enables GVL profiling. This will show when threads are waiting for GVL in the timeline view.
-            #
-            # This is a preview feature and disabled by default. It requires Ruby 3.2+.
-            #
-            # @default `DD_PROFILING_PREVIEW_GVL_ENABLED` environment variable as a boolean, otherwise `false`
+            # @deprecated Use {:gvl_enabled} instead.
             option :preview_gvl_enabled do |o|
               o.type :bool
-              o.env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
               o.default false
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Datadog.logger.warn(
+                    'The profiling.advanced.preview_gvl_enabled setting has been deprecated for removal and ' \
+                    'no longer does anything. Please remove it from your Datadog.configure block. ' \
+                    'GVL profiling is now controlled by the profiling.advanced.gvl_enabled setting instead.'
+                  )
+                end
+              end
+            end
+
+            # Controls GVL profiling. This will show when threads are waiting for GVL in the timeline view.
+            #
+            # This feature requires Ruby 3.2+.
+            #
+            # @default `DD_PROFILING_GVL_ENABLED` environment variable as a boolean, otherwise `true`
+            option :gvl_enabled do |o|
+              o.type :bool
+              o.deprecated_env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
+              o.env 'DD_PROFILING_GVL_ENABLED'
+              o.default true
             end
 
             # Controls the smallest time period the profiler will report a thread waiting for the GVL.

data/lib/datadog/core/encoding.rb

@@ -10,6 +10,7 @@ module Datadog
       # Encoder interface that provides the logic to encode traces and service
       # @abstract
       module Encoder
+        # :nocov:
         def content_type
           raise NotImplementedError
         end
@@ -23,6 +24,13 @@ module Datadog
         def encode(_)
           raise NotImplementedError
         end
+
+        # Deserializes a value serialized with {#encode}.
+        # This method is used for debugging purposes.
+        def decode(_)
+          raise NotImplementedError
+        end
+        # :nocov:
       end
 
       # Encoder for the JSON format
@@ -41,6 +49,10 @@ module Datadog
           JSON.dump(obj)
         end
 
+        def decode(obj)
+          JSON.parse(obj)
+        end
+
         def join(encoded_data)
           "[#{encoded_data.join(',')}]"
         end
@@ -62,6 +74,10 @@ module Datadog
           MessagePack.pack(obj)
         end
 
+        def decode(obj)
+          MessagePack.unpack(obj)
+        end
+
         def join(encoded_data)
           packer = MessagePack::Packer.new
           packer.write_array_header(encoded_data.size)

data/lib/datadog/core/environment/agent_info.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves the agent's `/info` endpoint data.
+      # This data can be used to determine the capabilities of the local Datadog agent.
+      #
+      # @example Example response payload
+      #   {
+      #     "version" : "7.57.2",
+      #     "git_commit" : "38ba0c7",
+      #     "endpoints" : [ "/v0.4/traces", "/v0.4/services", "/v0.7/traces", "/v0.7/config" ],
+      #     "client_drop_p0s" : true,
+      #     "span_meta_structs" : true,
+      #     "long_running_spans" : true,
+      #     "evp_proxy_allowed_headers" : [ "Content-Type", "Accept-Encoding", "Content-Encoding", "User-Agent" ],
+      #     "config" : {
+      #       "default_env" : "none",
+      #       "target_tps" : 10,
+      #       "max_eps" : 200,
+      #       "receiver_port" : 8126,
+      #       "receiver_socket" : "/var/run/datadog/apm.socket",
+      #       "connection_limit" : 0,
+      #       "receiver_timeout" : 0,
+      #       "max_request_bytes" : 26214400,
+      #       "statsd_port" : 8125,
+      #       "analyzed_spans_by_service" : { },
+      #       "obfuscation" : {
+      #         "elastic_search" : true,
+      #         "mongo" : true,
+      #         "sql_exec_plan" : false,
+      #         "sql_exec_plan_normalize" : false,
+      #         "http" : {
+      #           "remove_query_string" : false,
+      #           "remove_path_digits" : false
+      #         },
+      #         "remove_stack_traces" : false,
+      #         "redis" : {
+      #           "Enabled" : true,
+      #           "RemoveAllArgs" : false
+      #         },
+      #         "memcached" : {
+      #           "Enabled" : true,
+      #           "KeepCommand" : false
+      #         }
+      #       }
+      #     },
+      #     "peer_tags" : null
+      #   }
+      #
+      # @see https://github.com/DataDog/datadog-agent/blob/f07df0a3c1fca0c83b5a15f553bd994091b0c8ac/pkg/trace/api/info.go#L20
+      class AgentInfo
+        attr_reader :agent_settings
+
+        def initialize(agent_settings)
+          @agent_settings = agent_settings
+          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings)
+        end
+
+        # Fetches the information from the agent.
+        # @return [Datadog::Core::Remote::Transport::HTTP::Negotiation::Response] the response from the agent
+        # @return [nil] if an error occurred while fetching the information
+        def fetch
+          res = @client.send_info
+          return unless res.ok?
+
+          res
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && other.agent_settings == agent_settings
+        end
+      end
+    end
+  end
+end