datadog 2.4.0 → 2.5.0

data/lib/datadog/appsec/contrib/graphql/reactive/multiplex.rb

@@ -25,30 +25,17 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
                 arguments = values[0]
 
-                waf_args = {
+                persistent_data = {
                   'graphql.server.all_resolvers' => arguments
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
-
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                result = waf_context.run(persistent_data, {}, waf_timeout)
+
+                next if result.status != :match
+
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -41,11 +41,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end
@@ -85,11 +83,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end
@@ -129,11 +125,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -33,6 +33,7 @@ module Datadog
             def self.subscribe(op, waf_context)
               op.subscribe(*ADDRESSES) do |*values|
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+
                 headers = values[0]
                 headers_no_cookies = headers.dup.tap { |h| h.delete('cookie') }
                 uri_raw = values[1]
@@ -41,7 +42,7 @@ module Datadog
                 client_ip = values[4]
                 request_method = values[5]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.cookies' => cookies,
                   'server.request.query' => query,
                   'server.request.uri.raw' => uri_raw,
@@ -52,25 +53,12 @@ module Datadog
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
-
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+                result = waf_context.run(persistent_data, {}, waf_timeout)
 
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+                next if result.status != :match
 
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb

@@ -26,30 +26,17 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
                 body = values[0]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.body' => body,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
-
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                result = waf_context.run(persistent_data, {}, waf_timeout)
+
+                next if result.status != :match
+
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -30,32 +30,19 @@ module Datadog
                 response_headers = values[1]
                 response_headers_no_cookies = response_headers.dup.tap { |h| h.delete('set-cookie') }
 
-                waf_args = {
+                persistent_data = {
                   'server.response.status' => response_status.to_s,
                   'server.response.headers' => response_headers,
                   'server.response.headers.no_cookies' => response_headers_no_cookies,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(persistent_data, {}, waf_timeout)
 
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+                next if result.status != :match
 
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -150,7 +150,9 @@ module Datadog
 
             return unless trace && span
 
-            span.set_tag('_dd.appsec.enabled', 1)
+            span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
+            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
+            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -38,11 +38,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -31,31 +31,18 @@ module Datadog
                 body = values[0]
                 path_params = values[1]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.body' => body,
                   'server.request.path_params' => path_params,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(persistent_data, {}, waf_timeout)
 
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+                next if result.status != :match
 
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -40,11 +40,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end
@@ -84,11 +82,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -27,30 +27,17 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
                 path_params = values[0]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.path_params' => path_params,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
-
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                result = waf_context.run(persistent_data, {}, waf_timeout)
+
+                next if result.status != :match
+
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/event.rb

@@ -137,6 +137,18 @@ module Datadog
         end
         # rubocop:enable Metrics/MethodLength
 
+        def tag_and_keep!(scope, waf_result)
+          # We want to keep the trace in case of security event
+          scope.trace.keep! if scope.trace
+
+          if scope.service_entry_span
+            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.include?('block')
+            scope.service_entry_span.set_tag('appsec.event', 'true')
+          end
+
+          add_distributed_tags(scope.trace)
+        end
+
         private
 
         def compressed_and_base64_encoded(value)
@@ -165,6 +177,18 @@ module Datadog
           gz.close
           sio.string
         end
+
+        # Propagate to downstream services the information that the current distributed trace is
+        # containing at least one ASM security event.
+        def add_distributed_tags(trace)
+          return unless trace
+
+          trace.set_tag(
+            Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            Datadog::Tracing::Sampling::Ext::Decision::ASM
+          )
+          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+        end
       end
     end
   end

data/lib/datadog/appsec/ext.rb

@@ -5,6 +5,10 @@ module Datadog
     module Ext
       INTERRUPT = :datadog_appsec_interrupt
       SCOPE_KEY = 'datadog.appsec.scope'
+
+      TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
+      TAG_APM_ENABLED = '_dd.apm.enabled'
+      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
     end
   end
 end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -35,11 +35,9 @@ module Datadog
                         actions: result.actions
                       }
 
-                      if scope.service_entry_span
-                        scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                        scope.service_entry_span.set_tag('appsec.event', 'true')
-                      end
-
+                      # We want to keep the trace in case of security event
+                      scope.trace.keep! if scope.trace
+                      Datadog::AppSec::Event.tag_and_keep!(scope, result)
                       scope.processor_context.events << event
                     end
                   end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -25,30 +25,17 @@ module Datadog
 
               user_id = values[0]
 
-              waf_args = {
+              persistent_data = {
                 'usr.id' => user_id,
               }
 
               waf_timeout = Datadog.configuration.appsec.waf_timeout
-              result = waf_context.run(waf_args, waf_timeout)
-
-              Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-              case result.status
-              when :match
-                Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                yield result
-                throw(:block, true) unless result.actions.empty?
-              when :ok
-                Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-              when :invalid_call
-                Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-              when :invalid_rule, :invalid_flow, :no_rule
-                Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-              else
-                Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-              end
+              result = waf_context.run(persistent_data, {}, waf_timeout)
+
+              next if result.status != :match
+
+              yield result
+              throw(:block, true) unless result.actions.empty?
             end
           end
         end

data/lib/datadog/appsec/processor/context.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'libddwaf'
+
+module Datadog
+  module AppSec
+    class Processor
+      # Context manages a sequence of runs
+      class Context
+        LIBDDWAF_SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(handle, telemetry:)
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+          @run_mutex = Mutex.new
+
+          @libddwaf_debug_tag = "libddwaf:#{WAF::VERSION::STRING}"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # these updates are not thread safe and should be protected
+          @time_ns += result.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if result.timeout
+
+          report_execution(result)
+          result
+        ensure
+          @run_mutex.unlock
+        end
+
+        def extract_schema
+          return unless extract_schema?
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, result = try_run(input, {}, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          report_execution(result)
+          result
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@libddwaf_debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0.0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@libddwaf_debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if LIBDDWAF_SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@libddwaf_debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@libddwaf_debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -1,83 +1,19 @@
 # frozen_string_literal: true
 
+require_relative 'processor/context'
+
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
     class Processor
-      # Context manages a sequence of runs
-      class Context
-        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
-
-        def initialize(processor)
-          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
-          @time_ns = 0.0
-          @time_ext_ns = 0.0
-          @timeouts = 0
-          @events = []
-          @run_mutex = Mutex.new
-        end
-
-        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
-          @run_mutex.lock
-
-          start_ns = Core::Utils::Time.get_time(:nanosecond)
-
-          input.reject! do |_, v|
-            case v
-            when TrueClass, FalseClass
-              false
-            else
-              v.nil? ? true : v.empty?
-            end
-          end
-
-          _code, res = @context.run(input, timeout)
-
-          stop_ns = Core::Utils::Time.get_time(:nanosecond)
-
-          # these updates are not thread safe and should be protected
-          @time_ns += res.total_runtime
-          @time_ext_ns += (stop_ns - start_ns)
-          @timeouts += 1 if res.timeout
-
-          res
-        ensure
-          @run_mutex.unlock
-        end
-
-        def extract_schema
-          return unless extract_schema?
-
-          input = {
-            'waf.context.processor' => {
-              'extract-schema' => true
-            }
-          }
-
-          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
-
-          res
-        end
-
-        def finalize
-          @context.finalize
-        end
-
-        private
-
-        def extract_schema?
-          Datadog.configuration.appsec.api_security.enabled &&
-            Datadog.configuration.appsec.api_security.sample_rate.sample?
-        end
-      end
-
       attr_reader :diagnostics, :addresses
 
       def initialize(ruleset:, telemetry:)
+        @telemetry = telemetry
         @diagnostics = nil
         @addresses = []
+
         settings = Datadog.configuration.appsec
-        @telemetry = telemetry
 
         # TODO: Refactor to make it easier to test
         unless require_libddwaf && libddwaf_provides_waf? && create_waf_handle(settings, ruleset)
@@ -93,9 +29,9 @@ module Datadog
         @handle.finalize
       end
 
-      protected
-
-      attr_reader :handle
+      def new_context
+        Context.new(@handle, telemetry: @telemetry)
+      end
 
       private