datadog 2.15.0 → 2.16.0

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -4,9 +4,12 @@ require 'json'
 
 require_relative 'gateway/request'
 require_relative 'gateway/response'
-require_relative '../../instrumentation/gateway'
-require_relative '../../processor'
+
+require_relative '../../event'
 require_relative '../../response'
+require_relative '../../processor'
+require_relative '../../security_event'
+require_relative '../../instrumentation/gateway'
 
 require_relative '../../../tracing/client_ip'
 require_relative '../../../tracing/contrib/rack/header_collection'
@@ -36,7 +39,7 @@ module Datadog
             @rack_headers = {}
           end
 
-          # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          # rubocop:disable Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
@@ -97,20 +100,13 @@ module Datadog
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
             end
 
-            if AppSec.api_security_enabled?
-              ctx.events << {
-                trace: ctx.trace,
-                span: ctx.span,
-                waf_result: ctx.extract_schema,
-              }
-            end
-
-            ctx.events.each do |e|
-              e[:response] ||= gateway_response
-              e[:request]  ||= gateway_request
+            if AppSec.perform_api_security_check?
+              ctx.events.push(
+                AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
+              )
             end
 
-            AppSec::Event.record(ctx.span, *ctx.events)
+            AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)
 
             http_response
           ensure
@@ -119,7 +115,7 @@ module Datadog
               Datadog::AppSec::Context.deactivate
             end
           end
-          # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
+          # rubocop:enable Metrics/MethodLength
 
           private
 
@@ -143,6 +139,7 @@ module Datadog
             Datadog::Tracing.active_span
           end
 
+          # standard:disable Metrics/MethodLength
           def add_appsec_tags(processor, context)
             span = context.span
             trace = context.trace
@@ -177,7 +174,9 @@ module Datadog
               end
             end
           end
+          # standard:enable Metrics/MethodLength
 
+          # standard:disable Metrics/MethodLength
           def add_request_tags(context, env)
             span = context.span
 
@@ -200,6 +199,7 @@ module Datadog
               )
             end
           end
+          # standard:enable Metrics/MethodLength
 
           def to_rack_header(header)
             @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
 require_relative '../../../event'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
 
 module Datadog
   module AppSec
@@ -19,7 +20,7 @@ module Datadog
 
               def watch_request_action(gateway = Instrumentation.gateway)
                 gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.body' => gateway_request.parsed_body,
@@ -28,18 +29,15 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  if result.match? || !result.derivatives.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                  if result.match?
+                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -18,7 +18,7 @@ module Datadog
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
+            Gem.loaded_specs['railties']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -96,27 +96,27 @@ module Datadog
             # find tracer middleware reference in Rails::Configuration::MiddlewareStackProxy
             app.middleware.instance_variable_get(:@operations).each do |operation|
               args = case operation
-                     when Array
-                       # rails 5.2
-                       _op, args = operation
-                       args
-                     when Proc
-                       if operation.binding.local_variables.include?(:args)
-                         # rails 6.0, 6.1
-                         operation.binding.local_variable_get(:args)
-                       else
-                         # rails 7.0 uses ... to pass args
-                         args_getter = Class.new do
-                           def method_missing(_op, *args) # rubocop:disable Style/MissingRespondToMissing
-                             args
-                           end
-                         end.new
-                         operation.call(args_getter)
-                       end
-                     else
-                       # unknown, pass through
-                       []
-                     end
+              when Array
+                # rails 5.2
+                _op, args = operation
+                args
+              when Proc
+                if operation.binding.local_variables.include?(:args)
+                  # rails 6.0, 6.1
+                  operation.binding.local_variable_get(:args)
+                else
+                  # rails 7.0 uses ... to pass args
+                  args_getter = Class.new do
+                    def method_missing(_op, *args) # standard:disable Style/MissingRespondToMissing
+                      args
+                    end
+                  end.new
+                  operation.call(args_getter)
+                end
+              else
+                # unknown, pass through
+                []
+              end
 
               found = true if args.include?(middleware)
             end

data/lib/datadog/appsec/contrib/rest_client/integration.rb

@@ -20,7 +20,7 @@ module Datadog
           end
 
           def self.version
-            Gem.loaded_specs['rest-client'] && Gem.loaded_specs['rest-client'].version
+            Gem.loaded_specs['rest-client']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -1,6 +1,9 @@
 # rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -12,24 +15,20 @@ module Datadog
 
             context = AppSec.active_context
 
-            ephemeral_data = { 'server.io.net.url' => url }
+            ephemeral_data = {'server.io.net.url' => url}
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag_and_keep!(context, result)
 
-              context.events << {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                request_url: url,
-                actions: result.actions
-              }
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
 
-              ActionsHandler.handle(result.actions)
+              AppSec::ActionsHandler.handle(result.actions)
             end
 
-            super(&block)
+            super
           end
         end
       end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
 require_relative '../../../event'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
 
 module Datadog
   module AppSec
@@ -20,7 +21,7 @@ module Datadog
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.body' => gateway_request.form_hash
@@ -28,18 +29,15 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  if result.match? || !result.derivatives.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                  if result.match?
+                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)
@@ -48,7 +46,7 @@ module Datadog
 
               def watch_request_routed(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.path_params' => gateway_route_params.params
@@ -57,17 +55,13 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag_and_keep!(context, result)
 
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/sinatra/integration.rb

@@ -18,7 +18,7 @@ module Datadog
           register_as :sinatra
 
           def self.version
-            Gem.loaded_specs['sinatra'] && Gem.loaded_specs['sinatra'].version
+            Gem.loaded_specs['sinatra']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/event.rb

@@ -11,96 +11,106 @@ module Datadog
       DERIVATIVE_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
       DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
-        X-Forwarded-For
-        X-Client-IP
-        X-Real-IP
-        X-Forwarded
-        X-Cluster-Client-IP
-        Forwarded-For
-        Forwarded
-        Via
-        True-Client-IP
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-        Host
-        User-Agent
-        Accept
-        Accept-Encoding
-        Accept-Language
-      ].map!(&:downcase).freeze
+        x-forwarded-for
+        x-client-ip
+        x-real-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+        content-length
+        content-type
+        content-encoding
+        content-language
+        host
+        user-agent
+        accept
+        accept-encoding
+        accept-language
+      ].freeze
 
       ALLOWED_RESPONSE_HEADERS = %w[
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-      ].map!(&:downcase).freeze
-
-      # Record events for a trace
-      #
-      # This is expected to be called only once per trace for the rate limiter
-      # to properly apply
+        content-length
+        content-type
+        content-encoding
+        content-language
+      ].freeze
+
       class << self
-        def record(span, *events)
-          # ensure rate limiter is called only when there are events to record
-          return if events.empty? || span.nil?
+        def tag_and_keep!(context, waf_result)
+          # We want to keep the trace in case of security event
+          context.trace&.keep!
 
-          Datadog::AppSec::RateLimiter.thread_local.limit do
-            record_via_span(span, *events)
+          if context.span
+            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+              context.span.set_tag('appsec.blocked', 'true')
+            end
+
+            context.span.set_tag('appsec.event', 'true')
           end
+
+          add_distributed_tags(context.trace)
         end
 
-        def record_via_span(span, *events)
-          events.group_by { |e| e[:trace] }.each do |trace, event_group|
-            unless trace
-              Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
-              next
-            end
+        def record(context, request: nil, response: nil)
+          return if context.events.empty? || context.span.nil?
+
+          Datadog::AppSec::RateLimiter.thread_local.limit do
+            context.events.group_by(&:trace).each do |trace, event_group|
+              unless trace
+                next Datadog.logger.debug do
+                  "AppSec: Cannot record event group with #{event_group.count} events because it has no trace"
+                end
+              end
 
-            trace.keep!
-            trace.set_tag(
-              Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-              Datadog::Tracing::Sampling::Ext::Decision::ASM
-            )
+              if event_group.any? { |event| event.attack? || event.schema? }
+                trace.keep!
+                trace[Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER] = Tracing::Sampling::Ext::Decision::ASM
 
-            # prepare and gather tags to apply
-            service_entry_tags = build_service_entry_tags(event_group)
+                context.span['_dd.origin'] = 'appsec'
+                context.span.set_tags(request_tags(request)) if request
+                context.span.set_tags(response_tags(response)) if response
+              end
 
-            # apply tags to service entry span
-            service_entry_tags.each do |key, value|
-              span.set_tag(key, value)
+              context.span.set_tags(waf_tags(event_group))
             end
           end
         end
 
-        def build_service_entry_tags(event_group)
-          waf_events = []
-          entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
-            # TODO: assume HTTP request context for now
-            if (request = event[:request])
-              request.headers.each do |header, value|
-                tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
-              end
+        private
 
-              tags['http.host'] = request.host
-              tags['http.useragent'] = request.user_agent
-              tags['network.client.ip'] = request.remote_addr
-            end
+        def request_tags(request)
+          tags = {}
 
-            if (response = event[:response])
-              response.headers.each do |header, value|
-                tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
-              end
-            end
+          tags['http.host'] = request.host if request.host
+          tags['http.useragent'] = request.user_agent if request.user_agent
+          tags['network.client.ip'] = request.remote_addr if request.remote_addr
+
+          request.headers.each_with_object(tags) do |(name, value), memo|
+            next unless ALLOWED_REQUEST_HEADERS.include?(name)
+
+            memo["http.request.headers.#{name}"] = value
+          end
+        end
+
+        def response_tags(response)
+          response.headers.each_with_object({}) do |(name, value), memo|
+            next unless ALLOWED_RESPONSE_HEADERS.include?(name)
+
+            memo["http.response.headers.#{name}"] = value
+          end
+        end
 
-            waf_result = event[:waf_result]
-            # accumulate triggers
-            waf_events += waf_result.events
+        def waf_tags(security_events)
+          triggers = []
 
-            waf_result.derivatives.each do |key, value|
-              next tags[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
+          tags = security_events.each_with_object({}) do |security_event, memo|
+            triggers.concat(security_event.waf_result.events)
+
+            security_event.waf_result.derivatives.each do |key, value|
+              next memo[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
 
               value = CompressedJson.dump(value)
               next if value.nil?
@@ -110,34 +120,14 @@ module Datadog
                 next
               end
 
-              tags[key] = value
-            end
-
-            tags
-          end
-
-          appsec_events = json_parse({ triggers: waf_events })
-          entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
-          entry_tags
-        end
-
-        def tag_and_keep!(context, waf_result)
-          # We want to keep the trace in case of security event
-          context.trace.keep! if context.trace
-
-          if context.span
-            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
-              context.span.set_tag('appsec.blocked', 'true')
+              memo[key] = value
             end
-
-            context.span.set_tag('appsec.event', 'true')
           end
 
-          add_distributed_tags(context.trace)
+          tags['_dd.appsec.json'] = json_parse({triggers: triggers}) unless triggers.empty?
+          tags
         end
 
-        private
-
         # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
         #       the issue between Ruby encoded strings and libddwaf produced events and now
         #       is under investigation.

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -8,14 +8,17 @@ module Datadog
         class Argument; end # rubocop:disable Lint/EmptyClass
 
         # Gateway User argument
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
         class User < Argument
-          attr_reader :id, :login
+          attr_reader :id, :login, :session_id
 
-          def initialize(id, login)
+          def initialize(id, login = nil, session_id = nil)
             super()
 
             @id = id
             @login = login
+            @session_id = session_id
           end
         end
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -10,7 +10,7 @@ module Datadog
         def report_rasp(type, result)
           return if result.is_a?(SecurityEngine::Result::Error)
 
-          tags = { rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING }
+          tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
           AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)