datadog 2.12.1 → 2.19.0

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require 'json'
+
+require_relative '../../../event'
+require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
 module Datadog
@@ -29,17 +32,13 @@ module Datadog
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                     if result.match?
-                      Datadog::AppSec::Event.tag_and_keep!(context, result)
+                      AppSec::Event.tag_and_keep!(context, result)
 
-                      context.events << {
-                        waf_result: result,
-                        trace: context.trace,
-                        span: context.span,
-                        multiplex: gateway_multiplex,
-                        actions: result.actions
-                      }
+                      context.events.push(
+                        AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                      )
 
-                      Datadog::AppSec::ActionsHandler.handle(result.actions)
+                      AppSec::ActionsHandler.handle(result.actions)
                     end
                   end
 

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -23,7 +23,7 @@ module Datadog
           register_as :graphql, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version
+            Gem.loaded_specs['graphql']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -6,6 +6,40 @@ module Datadog
       module Rack
         # Rack integration constants
         module Ext
+          COLLECTABLE_REQUEST_HEADERS = [
+            'accept',
+            'akamai-user-risk',
+            'cf-ray',
+            'cloudfront-viewer-ja3-fingerprint',
+            'content-type',
+            'user-agent',
+            'x-amzn-trace-Id',
+            'x-appgw-trace-id',
+            'x-cloud-trace-context',
+            'x-sigsci-requestid',
+            'x-sigsci-tags'
+          ].freeze
+
+          IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
+            'accept-encoding',
+            'accept-language',
+            'cf-connecting-ip',
+            'cf-connecting-ipv6',
+            'content-encoding',
+            'content-language',
+            'content-length',
+            'fastly-client-ip',
+            'forwarded',
+            'forwarded-for',
+            'host',
+            'true-client-ip',
+            'via',
+            'x-client-ip',
+            'x-cluster-client-ip',
+            'x-forwarded',
+            'x-forwarded-for',
+            'x-real-ip'
+          ].freeze
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
+require_relative '../ext'
 require_relative '../../../event'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
 
 module Datadog
   module AppSec
@@ -17,11 +19,12 @@ module Datadog
                 watch_request(gateway)
                 watch_response(gateway)
                 watch_request_body(gateway)
+                watch_request_finish(gateway)
               end
 
               def watch_request(gateway = Instrumentation.gateway)
                 gateway.watch('rack.request', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.cookies' => gateway_request.cookies,
@@ -35,18 +38,15 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  if result.match? || !result.derivatives.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                  if result.match?
+                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)
@@ -66,17 +66,13 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag_and_keep!(context, result)
 
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      response: gateway_response,
-                      actions: result.actions
-                    }
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_response.response)
@@ -85,7 +81,7 @@ module Datadog
 
               def watch_request_body(gateway = Instrumentation.gateway)
                 gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.body' => gateway_request.form_hash
@@ -94,17 +90,38 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag_and_keep!(context, result)
+
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(gateway_request.request)
+                end
+              end
+
+              # NOTE: In the current state we unable to substibe twice to the same
+              #       event within the same group. Ideally this code should live
+              #       somewhere closer to identity related monitor.
+              # WARNING: The Gateway is a subject of refactoring
+              def watch_request_finish(gateway = Instrumentation.gateway)
+                gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+
+                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
+                    next stack.call(gateway_request.request)
+                  end
 
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  gateway_request.headers.each do |name, value|
+                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
+                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      next
+                    end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                    context.span["http.request.headers.#{name}"] ||= value
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -19,7 +19,7 @@ module Datadog
           register_as :rack, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['rack'] && Gem.loaded_specs['rack'].version
+            Gem.loaded_specs['rack']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -4,9 +4,12 @@ require 'json'
 
 require_relative 'gateway/request'
 require_relative 'gateway/response'
-require_relative '../../instrumentation/gateway'
-require_relative '../../processor'
+
+require_relative '../../event'
 require_relative '../../response'
+require_relative '../../api_security'
+require_relative '../../security_event'
+require_relative '../../instrumentation/gateway'
 
 require_relative '../../../tracing/client_ip'
 require_relative '../../../tracing/contrib/rack/header_collection'
@@ -36,14 +39,14 @@ module Datadog
             @rack_headers = {}
           end
 
-          # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          # rubocop:disable Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
             boot = Datadog::Core::Remote::Tie.boot
             Datadog::Core::Remote::Tie::Tracing.tag(boot, active_span)
 
-            processor = nil
+            security_engine = nil
             ready = false
             ctx = nil
 
@@ -53,11 +56,11 @@ module Datadog
             return @app.call(env) if active_context(env)
 
             Datadog::AppSec.reconfigure_lock do
-              processor = Datadog::AppSec.processor
+              security_engine = Datadog::AppSec.security_engine
 
-              if !processor.nil? && processor.ready?
+              if security_engine
                 ctx = Datadog::AppSec::Context.activate(
-                  Datadog::AppSec::Context.new(active_trace, active_span, processor)
+                  Datadog::AppSec::Context.new(active_trace, active_span, security_engine.new_runner)
                 )
 
                 env[Datadog::AppSec::Ext::CONTEXT_KEY] = ctx
@@ -69,7 +72,7 @@ module Datadog
 
             return @app.call(env) unless ready
 
-            add_appsec_tags(processor, ctx)
+            add_appsec_tags(security_engine, ctx)
             add_request_tags(ctx, env)
 
             http_response = nil
@@ -77,6 +80,8 @@ module Datadog
             gateway_response = nil
 
             interrupt_params = catch(::Datadog::AppSec::Ext::INTERRUPT) do
+              # TODO: This event should be renamed into `rack.request.start` to
+              #       reflect that it's the beginning of the request-cycle
               http_response, _gateway_request = Instrumentation.gateway.push('rack.request', gateway_request) do
                 @app.call(env)
               end
@@ -85,6 +90,7 @@ module Datadog
                 http_response[2], http_response[0], http_response[1], context: ctx
               )
 
+              Instrumentation.gateway.push('rack.request.finish', gateway_request)
               Instrumentation.gateway.push('rack.response', gateway_response)
 
               nil
@@ -94,20 +100,29 @@ module Datadog
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
             end
 
-            if AppSec.api_security_enabled?
-              ctx.events << {
-                trace: ctx.trace,
-                span: ctx.span,
-                waf_result: ctx.extract_schema,
-              }
+            # NOTE: This is not optimal, but in the current implementation
+            #       `gateway_response` is a container to dispatch response event
+            #       and in case of interruption it suppose to be `nil`.
+            #
+            #       `http_response` is a real response object in both cases, but
+            #       to save us some computations, we will use already pre-computed
+            #       `gateway_response` instead of re-creating it.
+            #
+            # WARNING: This part will be refactored.
+            tmp_response = if interrupt_params
+              Gateway::Response.new(http_response[2], http_response[0], http_response[1], context: ctx)
+            else
+              gateway_response
             end
 
-            ctx.events.each do |e|
-              e[:response] ||= gateway_response
-              e[:request]  ||= gateway_request
+            if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
+                AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
+              ctx.events.push(
+                AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
+              )
             end
 
-            AppSec::Event.record(ctx.span, *ctx.events)
+            AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)
 
             http_response
           ensure
@@ -116,7 +131,7 @@ module Datadog
               Datadog::AppSec::Context.deactivate
             end
           end
-          # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
+          # rubocop:enable Metrics/MethodLength
 
           private
 
@@ -140,32 +155,26 @@ module Datadog
             Datadog::Tracing.active_span
           end
 
-          def add_appsec_tags(processor, context)
+          # standard:disable Metrics/MethodLength
+          def add_appsec_tags(security_engine, context)
             span = context.span
             trace = context.trace
 
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            if processor.diagnostics
-              diagnostics = processor.diagnostics
-
-              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
+            if security_engine.ruleset_version
+              span.set_tag('_dd.appsec.event_rules.version', security_engine.ruleset_version)
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
-                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
+                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(security_engine.waf_addresses))
 
                 # Ensure these tags reach the backend
                 trace.keep!
@@ -176,7 +185,9 @@ module Datadog
               end
             end
           end
+          # standard:enable Metrics/MethodLength
 
+          # standard:disable Metrics/MethodLength
           def add_request_tags(context, env)
             span = context.span
 
@@ -199,6 +210,7 @@ module Datadog
               )
             end
           end
+          # standard:enable Metrics/MethodLength
 
           def to_rack_header(header)
             @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
 require_relative '../../../event'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
 
 module Datadog
   module AppSec
@@ -19,7 +20,7 @@ module Datadog
 
               def watch_request_action(gateway = Instrumentation.gateway)
                 gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.body' => gateway_request.parsed_body,
@@ -28,18 +29,15 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  if result.match? || !result.derivatives.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                  if result.match?
+                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -18,7 +18,7 @@ module Datadog
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
+            Gem.loaded_specs['railties']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -96,27 +96,27 @@ module Datadog
             # find tracer middleware reference in Rails::Configuration::MiddlewareStackProxy
             app.middleware.instance_variable_get(:@operations).each do |operation|
               args = case operation
-                     when Array
-                       # rails 5.2
-                       _op, args = operation
-                       args
-                     when Proc
-                       if operation.binding.local_variables.include?(:args)
-                         # rails 6.0, 6.1
-                         operation.binding.local_variable_get(:args)
-                       else
-                         # rails 7.0 uses ... to pass args
-                         args_getter = Class.new do
-                           def method_missing(_op, *args) # rubocop:disable Style/MissingRespondToMissing
-                             args
-                           end
-                         end.new
-                         operation.call(args_getter)
-                       end
-                     else
-                       # unknown, pass through
-                       []
-                     end
+              when Array
+                # rails 5.2
+                _op, args = operation
+                args
+              when Proc
+                if operation.binding.local_variables.include?(:args)
+                  # rails 6.0, 6.1
+                  operation.binding.local_variable_get(:args)
+                else
+                  # rails 7.0 uses ... to pass args
+                  args_getter = Class.new do
+                    def method_missing(_op, *args) # standard:disable Style/MissingRespondToMissing
+                      args
+                    end
+                  end.new
+                  operation.call(args_getter)
+                end
+              else
+                # unknown, pass through
+                []
+              end
 
               found = true if args.include?(middleware)
             end

data/lib/datadog/appsec/contrib/rest_client/integration.rb

@@ -20,7 +20,7 @@ module Datadog
           end
 
           def self.version
-            Gem.loaded_specs['rest-client'] && Gem.loaded_specs['rest-client'].version
+            Gem.loaded_specs['rest-client']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -1,6 +1,9 @@
 # rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -12,24 +15,20 @@ module Datadog
 
             context = AppSec.active_context
 
-            ephemeral_data = { 'server.io.net.url' => url }
+            ephemeral_data = {'server.io.net.url' => url}
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag_and_keep!(context, result)
 
-              context.events << {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                request_url: url,
-                actions: result.actions
-              }
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
 
-              ActionsHandler.handle(result.actions)
+              AppSec::ActionsHandler.handle(result.actions)
             end
 
-            super(&block)
+            super
           end
         end
       end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
 require_relative '../../../event'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
 
 module Datadog
   module AppSec
@@ -20,7 +21,7 @@ module Datadog
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.body' => gateway_request.form_hash
@@ -28,18 +29,15 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                  if result.match? || !result.derivatives.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                  if result.match?
+                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)
@@ -48,7 +46,7 @@ module Datadog
 
               def watch_request_routed(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
-                  context = gateway_request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
                     'server.request.path_params' => gateway_route_params.params
@@ -57,17 +55,13 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag_and_keep!(context, result)
 
-                    context.events << {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      request: gateway_request,
-                      actions: result.actions
-                    }
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
 
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/sinatra/integration.rb

@@ -18,7 +18,7 @@ module Datadog
           register_as :sinatra
 
           def self.version
-            Gem.loaded_specs['sinatra'] && Gem.loaded_specs['sinatra'].version
+            Gem.loaded_specs['sinatra']&.version
           end
 
           def self.loaded?