datadog 2.3.0 → 2.5.0

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -31,31 +31,18 @@ module Datadog
                 body = values[0]
                 path_params = values[1]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.body' => body,
                   'server.request.path_params' => path_params,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(persistent_data, {}, waf_timeout)
 
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+                next if result.status != :match
 
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -40,11 +40,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end
@@ -84,11 +82,9 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        # We want to keep the trace in case of security event
+                        scope.trace.keep! if scope.trace
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end
                     end

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -27,30 +27,17 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
                 path_params = values[0]
 
-                waf_args = {
+                persistent_data = {
                   'server.request.path_params' => path_params,
                 }
 
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
-                result = waf_context.run(waf_args, waf_timeout)
-
-                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-                case result.status
-                when :match
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                  yield result
-                  throw(:block, true) unless result.actions.empty?
-                when :ok
-                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-                when :invalid_call
-                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-                when :invalid_rule, :invalid_flow, :no_rule
-                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-                else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-                end
+                result = waf_context.run(persistent_data, {}, waf_timeout)
+
+                next if result.status != :match
+
+                yield result
+                throw(:block, true) unless result.actions.empty?
               end
             end
           end

data/lib/datadog/appsec/event.rb

@@ -52,7 +52,7 @@ module Datadog
           # ensure rate limiter is called only when there are events to record
           return if events.empty? || span.nil?
 
-          Datadog::AppSec::RateLimiter.limit(:traces) do
+          Datadog::AppSec::RateLimiter.thread_local.limit do
             record_via_span(span, *events)
           end
         end
@@ -137,6 +137,18 @@ module Datadog
         end
         # rubocop:enable Metrics/MethodLength
 
+        def tag_and_keep!(scope, waf_result)
+          # We want to keep the trace in case of security event
+          scope.trace.keep! if scope.trace
+
+          if scope.service_entry_span
+            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.include?('block')
+            scope.service_entry_span.set_tag('appsec.event', 'true')
+          end
+
+          add_distributed_tags(scope.trace)
+        end
+
         private
 
         def compressed_and_base64_encoded(value)
@@ -165,6 +177,18 @@ module Datadog
           gz.close
           sio.string
         end
+
+        # Propagate to downstream services the information that the current distributed trace is
+        # containing at least one ASM security event.
+        def add_distributed_tags(trace)
+          return unless trace
+
+          trace.set_tag(
+            Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            Datadog::Tracing::Sampling::Ext::Decision::ASM
+          )
+          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+        end
       end
     end
   end

data/lib/datadog/appsec/ext.rb

@@ -5,6 +5,10 @@ module Datadog
     module Ext
       INTERRUPT = :datadog_appsec_interrupt
       SCOPE_KEY = 'datadog.appsec.scope'
+
+      TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
+      TAG_APM_ENABLED = '_dd.apm.enabled'
+      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
     end
   end
 end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -35,11 +35,9 @@ module Datadog
                         actions: result.actions
                       }
 
-                      if scope.service_entry_span
-                        scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                        scope.service_entry_span.set_tag('appsec.event', 'true')
-                      end
-
+                      # We want to keep the trace in case of security event
+                      scope.trace.keep! if scope.trace
+                      Datadog::AppSec::Event.tag_and_keep!(scope, result)
                       scope.processor_context.events << event
                     end
                   end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -25,30 +25,17 @@ module Datadog
 
               user_id = values[0]
 
-              waf_args = {
+              persistent_data = {
                 'usr.id' => user_id,
               }
 
               waf_timeout = Datadog.configuration.appsec.waf_timeout
-              result = waf_context.run(waf_args, waf_timeout)
-
-              Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
-
-              case result.status
-              when :match
-                Datadog.logger.debug { "WAF: #{result.inspect}" }
-
-                yield result
-                throw(:block, true) unless result.actions.empty?
-              when :ok
-                Datadog.logger.debug { "WAF OK: #{result.inspect}" }
-              when :invalid_call
-                Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
-              when :invalid_rule, :invalid_flow, :no_rule
-                Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
-              else
-                Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
-              end
+              result = waf_context.run(persistent_data, {}, waf_timeout)
+
+              next if result.status != :match
+
+              yield result
+              throw(:block, true) unless result.actions.empty?
             end
           end
         end

data/lib/datadog/appsec/processor/context.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'libddwaf'
+
+module Datadog
+  module AppSec
+    class Processor
+      # Context manages a sequence of runs
+      class Context
+        LIBDDWAF_SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(handle, telemetry:)
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+          @run_mutex = Mutex.new
+
+          @libddwaf_debug_tag = "libddwaf:#{WAF::VERSION::STRING}"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # these updates are not thread safe and should be protected
+          @time_ns += result.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if result.timeout
+
+          report_execution(result)
+          result
+        ensure
+          @run_mutex.unlock
+        end
+
+        def extract_schema
+          return unless extract_schema?
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, result = try_run(input, {}, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          report_execution(result)
+          result
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@libddwaf_debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0.0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@libddwaf_debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if LIBDDWAF_SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@libddwaf_debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@libddwaf_debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -9,7 +9,7 @@ module Datadog
       # that load appsec rules and data from  settings
       module RuleLoader
         class << self
-          def load_rules(ruleset:)
+          def load_rules(ruleset:, telemetry:)
             begin
               case ruleset
               when :recommended, :strict
@@ -35,6 +35,8 @@ module Datadog
                 "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
               end
 
+              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+
               nil
             end
           end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -18,25 +18,35 @@ module Datadog
           end
         end
 
-        DEFAULT_WAF_PROCESSORS = begin
-          JSON.parse(Datadog::AppSec::Assets.waf_processors)
-        rescue StandardError => e
-          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
-          []
-        end
-
-        DEFAULT_WAF_SCANNERS = begin
-          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
-        rescue StandardError => e
-          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
-          []
-        end
-
         class << self
+          # TODO: `processors` and `scanners` are not provided by the caller, consider removing them
           def merge(
+            telemetry:,
             rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
-            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
+            processors: nil, scanners: nil
           )
+            processors ||= begin
+              default_waf_processors
+            rescue StandardError => e
+              Datadog.logger.error("libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}")
+              telemetry.report(
+                e,
+                description: 'libddwaf rulemerger failed to parse default waf processors'
+              )
+              []
+            end
+
+            scanners ||= begin
+              default_waf_scanners
+            rescue StandardError => e
+              Datadog.logger.error("libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}")
+              telemetry.report(
+                e,
+                description: 'libddwaf rulemerger failed to parse default waf scanners'
+              )
+              []
+            end
+
             combined_rules = combine_rules(rules)
 
             combined_data = combine_data(data) if data.any?
@@ -53,6 +63,14 @@ module Datadog
             combined_rules
           end
 
+          def default_waf_processors
+            @default_waf_processors ||= JSON.parse(Datadog::AppSec::Assets.waf_processors)
+          end
+
+          def default_waf_scanners
+            @default_waf_scanners ||= JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+          end
+
           private
 
           def combine_rules(rules)

data/lib/datadog/appsec/processor.rb

@@ -1,85 +1,23 @@
 # frozen_string_literal: true
 
+require_relative 'processor/context'
+
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
     class Processor
-      # Context manages a sequence of runs
-      class Context
-        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
-
-        def initialize(processor)
-          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
-          @time_ns = 0.0
-          @time_ext_ns = 0.0
-          @timeouts = 0
-          @events = []
-          @run_mutex = Mutex.new
-        end
-
-        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
-          @run_mutex.lock
-
-          start_ns = Core::Utils::Time.get_time(:nanosecond)
-
-          input.reject! do |_, v|
-            case v
-            when TrueClass, FalseClass
-              false
-            else
-              v.nil? ? true : v.empty?
-            end
-          end
-
-          _code, res = @context.run(input, timeout)
-
-          stop_ns = Core::Utils::Time.get_time(:nanosecond)
-
-          # these updates are not thread safe and should be protected
-          @time_ns += res.total_runtime
-          @time_ext_ns += (stop_ns - start_ns)
-          @timeouts += 1 if res.timeout
-
-          res
-        ensure
-          @run_mutex.unlock
-        end
-
-        def extract_schema
-          return unless extract_schema?
-
-          input = {
-            'waf.context.processor' => {
-              'extract-schema' => true
-            }
-          }
-
-          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
-
-          res
-        end
-
-        def finalize
-          @context.finalize
-        end
-
-        private
-
-        def extract_schema?
-          Datadog.configuration.appsec.api_security.enabled &&
-            Datadog.configuration.appsec.api_security.sample_rate.sample?
-        end
-      end
-
       attr_reader :diagnostics, :addresses
 
-      def initialize(ruleset:)
+      def initialize(ruleset:, telemetry:)
+        @telemetry = telemetry
         @diagnostics = nil
         @addresses = []
+
         settings = Datadog.configuration.appsec
 
-        unless load_libddwaf && create_waf_handle(settings, ruleset)
-          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+        # TODO: Refactor to make it easier to test
+        unless require_libddwaf && libddwaf_provides_waf? && create_waf_handle(settings, ruleset)
+          Datadog.logger.warn('AppSec is disabled, see logged errors above')
         end
       end
 
@@ -91,14 +29,33 @@ module Datadog
         @handle.finalize
       end
 
-      protected
-
-      attr_reader :handle
+      def new_context
+        Context.new(@handle, telemetry: @telemetry)
+      end
 
       private
 
-      def load_libddwaf
-        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      # libddwaf raises a LoadError on unsupported platforms; it may at some
+      # point succeed in being required yet not provide a specific needed feature.
+      def require_libddwaf
+        Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+        require 'libddwaf'
+
+        true
+      rescue LoadError => e
+        Datadog.logger.error do
+          'libddwaf failed to load,' \
+            "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+        end
+        @telemetry.report(e, description: 'libddwaf failed to load')
+
+        false
+      end
+
+      # check whether libddwaf is required *and* able to provide the needed feature
+      def libddwaf_provides_waf?
+        defined?(Datadog::AppSec::WAF) ? true : false
       end
 
       def create_waf_handle(settings, ruleset)
@@ -119,6 +76,7 @@ module Datadog
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
+        @telemetry.report(e, description: 'libddwaf failed to initialize')
 
         @diagnostics = e.diagnostics if e.diagnostics
 
@@ -127,44 +85,21 @@ module Datadog
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
+        @telemetry.report(e, description: 'libddwaf failed to initialize')
 
         false
       end
 
-      class << self
-        # check whether libddwaf is required *and* able to provide the needed feature
-        def libddwaf_provides_waf?
-          defined?(Datadog::AppSec::WAF) ? true : false
-        end
-
-        # libddwaf raises a LoadError on unsupported platforms; it may at some
-        # point succeed in being required yet not provide a specific needed feature.
-        def require_libddwaf
-          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
-
-          require 'libddwaf'
-
-          true
-        rescue LoadError => e
-          Datadog.logger.error do
-            'libddwaf failed to load,' \
-              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
-          end
-
-          false
-        end
-
-        def libddwaf_spec
-          Gem.loaded_specs['libddwaf']
-        end
-
-        def libddwaf_platform
-          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+      def libddwaf_platform
+        if Gem.loaded_specs['libddwaf']
+          Gem.loaded_specs['libddwaf'].platform.to_s
+        else
+          'unknown'
         end
+      end
 
-        def ruby_platforms
-          Gem.platforms.map(&:to_s)
-        end
+      def ruby_platforms
+        Gem.platforms.map(&:to_s)
       end
     end
   end