datadog 2.12.1 → 2.19.0

data/lib/datadog/appsec/event.rb

@@ -1,181 +1,142 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'zlib'
-
 require_relative 'rate_limiter'
-require_relative '../core/utils/base64'
+require_relative 'compressed_json'
 
 module Datadog
   module AppSec
     # AppSec event
     module Event
+      DERIVATIVE_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
-        X-Forwarded-For
-        X-Client-IP
-        X-Real-IP
-        X-Forwarded
-        X-Cluster-Client-IP
-        Forwarded-For
-        Forwarded
-        Via
-        True-Client-IP
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-        Host
-        User-Agent
-        Accept
-        Accept-Encoding
-        Accept-Language
-      ].map!(&:downcase).freeze
+        x-forwarded-for
+        x-client-ip
+        x-real-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+        content-length
+        content-type
+        content-encoding
+        content-language
+        host
+        user-agent
+        accept
+        accept-encoding
+        accept-language
+      ].freeze
 
       ALLOWED_RESPONSE_HEADERS = %w[
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-      ].map!(&:downcase).freeze
-
-      MAX_ENCODED_SCHEMA_SIZE = 25000
-      # For more information about this number
-      # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
-      MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
-
-      # Record events for a trace
-      #
-      # This is expected to be called only once per trace for the rate limiter
-      # to properly apply
-      class << self
-        def record(span, *events)
-          # ensure rate limiter is called only when there are events to record
-          return if events.empty? || span.nil?
+        content-length
+        content-type
+        content-encoding
+        content-language
+      ].freeze
 
-          Datadog::AppSec::RateLimiter.thread_local.limit do
-            record_via_span(span, *events)
-          end
-        end
+      class << self
+        def tag_and_keep!(context, waf_result)
+          # We want to keep the trace in case of security event
+          context.trace&.keep!
 
-        def record_via_span(span, *events)
-          events.group_by { |e| e[:trace] }.each do |trace, event_group|
-            unless trace
-              Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
-              next
+          if context.span
+            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+              context.span.set_tag('appsec.blocked', 'true')
             end
 
-            trace.keep!
-            trace.set_tag(
-              Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-              Datadog::Tracing::Sampling::Ext::Decision::ASM
-            )
-
-            # prepare and gather tags to apply
-            service_entry_tags = build_service_entry_tags(event_group)
-
-            # apply tags to service entry span
-            service_entry_tags.each do |key, value|
-              span.set_tag(key, value)
-            end
+            context.span.set_tag('appsec.event', 'true')
           end
+
+          add_distributed_tags(context.trace)
         end
 
-        # rubocop:disable Metrics/MethodLength
-        def build_service_entry_tags(event_group)
-          waf_events = []
-          entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
-            # TODO: assume HTTP request context for now
-            if (request = event[:request])
-              request.headers.each do |header, value|
-                tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
+        def record(context, request: nil, response: nil)
+          return if context.events.empty? || context.span.nil?
+
+          Datadog::AppSec::RateLimiter.thread_local.limit do
+            context.events.group_by(&:trace).each do |trace, event_group|
+              unless trace
+                next Datadog.logger.debug do
+                  "AppSec: Cannot record event group with #{event_group.count} events because it has no trace"
+                end
               end
 
-              tags['http.host'] = request.host
-              tags['http.useragent'] = request.user_agent
-              tags['network.client.ip'] = request.remote_addr
-            end
+              if event_group.any? { |event| event.attack? || event.schema? }
+                trace.keep!
+                trace[Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER] = Tracing::Sampling::Ext::Decision::ASM
 
-            if (response = event[:response])
-              response.headers.each do |header, value|
-                tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
+                context.span['_dd.origin'] = 'appsec'
+                context.span.set_tags(request_tags(request)) if request
+                context.span.set_tags(response_tags(response)) if response
               end
+
+              context.span.set_tags(waf_tags(event_group))
             end
+          end
+        end
 
-            waf_result = event[:waf_result]
-            # accumulate triggers
-            waf_events += waf_result.events
+        private
 
-            waf_result.derivatives.each do |key, value|
-              parsed_value = json_parse(value)
-              next unless parsed_value
+        def request_tags(request)
+          tags = {}
 
-              parsed_value_size = parsed_value.size
+          tags['http.host'] = request.host if request.host
+          tags['http.useragent'] = request.user_agent if request.user_agent
+          tags['network.client.ip'] = request.remote_addr if request.remote_addr
 
-              schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
-                               compressed_and_base64_encoded(parsed_value)
-                             else
-                               parsed_value
-                             end
-              next unless schema_value
+          request.headers.each_with_object(tags) do |(name, value), memo|
+            next unless ALLOWED_REQUEST_HEADERS.include?(name)
 
-              if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
-                Datadog.logger.debug do
-                  "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
-                end
-                next
-              end
+            memo["http.request.headers.#{name}"] = value
+          end
+        end
 
-              tags[key] = schema_value
-            end
+        def response_tags(response)
+          response.headers.each_with_object({}) do |(name, value), memo|
+            next unless ALLOWED_RESPONSE_HEADERS.include?(name)
 
-            tags
+            memo["http.response.headers.#{name}"] = value
           end
-
-          appsec_events = json_parse({ triggers: waf_events })
-          entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
-          entry_tags
         end
-        # rubocop:enable Metrics/MethodLength
 
-        def tag_and_keep!(context, waf_result)
-          # We want to keep the trace in case of security event
-          context.trace.keep! if context.trace
+        def waf_tags(security_events)
+          triggers = []
 
-          if context.span
-            context.span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
-            context.span.set_tag('appsec.event', 'true')
-          end
+          tags = security_events.each_with_object({}) do |security_event, memo|
+            triggers.concat(security_event.waf_result.events)
 
-          add_distributed_tags(context.trace)
-        end
+            security_event.waf_result.derivatives.each do |key, value|
+              next memo[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
 
-        private
+              value = CompressedJson.dump(value)
+              next if value.nil?
+
+              if value.size >= DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE
+                Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
+                next
+              end
 
-        def compressed_and_base64_encoded(value)
-          Datadog::Core::Utils::Base64.strict_encode64(gzip(value))
-        rescue TypeError => e
-          Datadog.logger.debug do
-            "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
+              memo[key] = value
+            end
           end
-          nil
+
+          tags['_dd.appsec.json'] = json_parse({triggers: triggers}) unless triggers.empty?
+          tags
         end
 
+        # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
+        #       the issue between Ruby encoded strings and libddwaf produced events and now
+        #       is under investigation.
         def json_parse(value)
           JSON.dump(value)
-        rescue ArgumentError => e
-          Datadog.logger.debug do
-            "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
+        rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+          AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
 
-        def gzip(value)
-          sio = StringIO.new
-          # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
-          gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
-          gz.write(value)
-          gz.close
-          sio.string
+          nil
         end
 
         # Propagate to downstream services the information that the current distributed trace is
@@ -187,7 +148,7 @@ module Datadog
             Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
             Datadog::Tracing::Sampling::Ext::Decision::ASM
           )
-          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+          trace.set_distributed_source(Datadog::AppSec::Ext::PRODUCT_BIT)
         end
       end
     end

data/lib/datadog/appsec/ext.rb

@@ -7,13 +7,15 @@ module Datadog
       RASP_LFI = 'lfi'
       RASP_SSRF = 'ssrf'
 
+      PRODUCT_BIT = 0b00000010
+
       INTERRUPT = :datadog_appsec_interrupt
       CONTEXT_KEY = 'datadog.appsec.context'
       ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
+      EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit'
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
-      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
+      TAG_METASTRUCT_STACK_TRACE = '_dd.stack'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'
     end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -8,12 +8,17 @@ module Datadog
         class Argument; end # rubocop:disable Lint/EmptyClass
 
         # Gateway User argument
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
         class User < Argument
-          attr_reader :id
+          attr_reader :id, :login, :session_id
 
-          def initialize(id)
+          def initialize(id = nil, login = nil, session_id = nil)
             super()
+
             @id = id
+            @login = login
+            @session_id = session_id
           end
         end
       end

data/lib/datadog/appsec/instrumentation/gateway/middleware.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Instrumentation
+      class Gateway
+        # NOTE: This class extracted as-is and will be deprecated
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(stack, env)
+            @block.call(stack, env)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,35 +1,29 @@
 # frozen_string_literal: true
 
+require_relative 'gateway/middleware'
+
 module Datadog
   module AppSec
     # Instrumentation for AppSec
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
-        # Instrumentation gateway middleware
-        class Middleware
-          attr_reader :key, :block
-
-          def initialize(key, &block)
-            @key = key
-            @block = block
-          end
-
-          def call(stack, env)
-            @block.call(stack, env)
-          end
-        end
-
-        private_constant :Middleware
-
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
+          @pushed_events = {}
         end
 
+        # NOTE: Be careful with pushed names because every pushed event name
+        #       is recorded in order to provide an ability to any subscriber
+        #       to check wether an arbitrary event had happened.
+        #
+        # WARNING: If we start pushing generated names we should consider
+        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          block ||= -> {}
+          @pushed_events[name] = true
 
-          middlewares_for_name = middlewares[name]
+          block ||= -> {}
+          middlewares_for_name = @middlewares[name]
 
           return [block.call, nil] if middlewares_for_name.empty?
 
@@ -48,14 +42,15 @@ module Datadog
         end
 
         def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
         end
 
-        private
-
-        attr_reader :middlewares
+        def pushed?(name)
+          @pushed_events.key?(name)
+        end
       end
 
+      # NOTE: This left as-is and will be depricated soon.
       def self.gateway
         @gateway ||= Gateway.new # TODO: not thread safe
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -10,7 +10,7 @@ module Datadog
         def report_rasp(type, result)
           return if result.is_a?(SecurityEngine::Result::Error)
 
-          tags = { rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING }
+          tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
           AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
 module Datadog
@@ -8,38 +10,71 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
+          ARBITRARY_VALUE = 'invalid'
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
           class << self
             def watch
               gateway = Instrumentation.gateway
 
               watch_user_id(gateway)
+              watch_user_login(gateway)
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                context = Datadog::AppSec.active_context
+                context = AppSec.active_context
+
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
 
-                persistent_data = {
-                  'usr.id' => user.id
-                }
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
+                persistent_data['usr.session_id'] = user.session_id if user.session_id
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
                 if result.match?
-                  Datadog::AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
 
-                  context.events << {
-                    waf_result: result,
-                    trace: context.trace,
-                    span: context.span,
-                    user: user,
-                    actions: result.actions
-                  }
+                stack.call(user)
+              end
+            end
 
-                  Datadog::AppSec::ActionsHandler.handle(result.actions)
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+                context = AppSec.active_context
+
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
                 end
 
-                stack.call(user)
+                if result.match?
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(kind)
               end
             end
           end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -10,50 +10,47 @@ module Datadog
       module RuleLoader
         class << self
           def load_rules(ruleset:, telemetry:)
-            begin
-              case ruleset
-              when :recommended, :strict
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
-              when :risky
-                Datadog.logger.warn(
-                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                  'The `:recommended` ruleset will be used instead.'\
-                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                )
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-              when String
-                JSON.parse(File.read(File.expand_path(ruleset)))
-              when File, StringIO
-                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
-              when Hash
-                ruleset
-              else
-                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
-              end
-            rescue StandardError => e
-              Datadog.logger.error do
-                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
-              end
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
+                'The `:recommended` ruleset will be used instead.' \
+                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
 
-              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: 'libddwaf ruleset failed to load')
 
-              nil
-            end
+            raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
-            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
+            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
 
             data
           end
 
           def load_exclusions(ip_passlist: [])
-            exclusions = []
-            exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
+            return [] if ip_passlist.empty?
 
-            exclusions
+            passlist_exclusions(ip_passlist)
           end
 
           private
@@ -62,7 +59,7 @@ module Datadog
             {
               'id' => id,
               'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
             }
           end