datadog 2.18.0 → 2.20.0

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -8,6 +8,7 @@ require_relative 'framework'
 require_relative 'gateway/watcher'
 require_relative 'gateway/route_params'
 require_relative 'gateway/request'
+require_relative 'patches/json_patch'
 require_relative '../../../tracing/contrib/sinatra/framework'
 
 module Datadog
@@ -51,7 +52,6 @@ module Datadog
         module DispatchPatch
           def dispatch!
             env = @request.env
-
             context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
 
             return super unless context
@@ -59,7 +59,6 @@ module Datadog
             # TODO: handle exceptions, except for super
 
             gateway_request = Gateway::Request.new(env)
-
             request_return, _gateway_request = Instrumentation.gateway.push('sinatra.request.dispatch', gateway_request) do
               super
             end
@@ -113,27 +112,18 @@ module Datadog
 
           def patch
             Gateway::Watcher.watch
-            patch_default_middlewares
-            patch_dispatch
-            patch_route
-            setup_security
-            Patcher.instance_variable_set(:@patched, true)
-          end
-
-          def setup_security
-            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
-          end
 
-          def patch_default_middlewares
             ::Sinatra::Base.singleton_class.prepend(DefaultMiddlewarePatch)
-          end
-
-          def patch_dispatch
             ::Sinatra::Base.prepend(DispatchPatch)
+            ::Sinatra::Base.prepend(RoutePatch)
+            ::Sinatra::Base.prepend(Patches::JsonPatch) if patch_json?
+            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
 
-          def patch_route
-            ::Sinatra::Base.prepend(RoutePatch)
+          def patch_json?
+            defined?(::Sinatra::JSON) && ::Sinatra::Base < ::Sinatra::JSON
           end
         end
       end

data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../../../utils/hash_coercion'
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        module Patches
+          # A patch targeting `Sinatra::JSON#json` method to capture JSON response
+          # body right before it is serialized.
+          module JsonPatch
+            def json(object, options = {})
+              context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              return super unless context
+
+              data = Utils::HashCoercion.coerce(object)
+              return super unless data
+
+              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              Instrumentation.gateway.push('sinatra.response.body.json', container)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/event.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require_relative 'rate_limiter'
+require_relative 'trace_keeper'
 require_relative 'compressed_json'
 
 module Datadog
@@ -40,8 +41,7 @@ module Datadog
 
       class << self
         def tag_and_keep!(context, waf_result)
-          # We want to keep the trace in case of security event
-          context.trace&.keep!
+          TraceKeeper.keep!(context.trace)
 
           if context.span
             if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
@@ -50,8 +50,6 @@ module Datadog
 
             context.span.set_tag('appsec.event', 'true')
           end
-
-          add_distributed_tags(context.trace)
         end
 
         def record(context, request: nil, response: nil)
@@ -66,8 +64,7 @@ module Datadog
               end
 
               if event_group.any? { |event| event.attack? || event.schema? }
-                trace.keep!
-                trace[Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER] = Tracing::Sampling::Ext::Decision::ASM
+                TraceKeeper.keep!(trace)
 
                 context.span['_dd.origin'] = 'appsec'
                 context.span.set_tags(request_tags(request)) if request
@@ -138,18 +135,6 @@ module Datadog
 
           nil
         end
-
-        # Propagate to downstream services the information that the current distributed trace is
-        # containing at least one ASM security event.
-        def add_distributed_tags(trace)
-          return unless trace
-
-          trace.set_tag(
-            Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-            Datadog::Tracing::Sampling::Ext::Decision::ASM
-          )
-          trace.set_distributed_source(Datadog::AppSec::Ext::PRODUCT_BIT)
-        end
       end
     end
   end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -13,7 +13,7 @@ module Datadog
         class User < Argument
           attr_reader :id, :login, :session_id
 
-          def initialize(id, login = nil, session_id = nil)
+          def initialize(id = nil, login = nil, session_id = nil)
             super()
 
             @id = id
@@ -21,6 +21,22 @@ module Datadog
             @session_id = session_id
           end
         end
+
+        # This class is used to pass arbitrary data to the event system with an
+        # option to tie it to a context.
+        #
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
+        class DataContainer < Argument
+          attr_reader :data, :context
+
+          def initialize(data, context:)
+            super()
+
+            @data = data
+            @context = context
+          end
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics/collector.rb

@@ -5,19 +5,21 @@ module Datadog
     module Metrics
       # A class responsible for collecting WAF and RASP call metrics.
       class Collector
-        Store = Struct.new(:evals, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
+        Store = Struct.new(:evals, :matches, :errors, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
 
         attr_reader :waf, :rasp
 
         def initialize
           @mutex = Mutex.new
-          @waf = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
-          @rasp = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+          @waf = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+          @rasp = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
         end
 
         def record_waf(result)
           @mutex.synchronize do
             @waf.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @waf.timeouts += 1 if result.timeout?
             @waf.duration_ns += result.duration_ns
             @waf.duration_ext_ns += result.duration_ext_ns
@@ -27,6 +29,8 @@ module Datadog
         def record_rasp(result)
           @mutex.synchronize do
             @rasp.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @rasp.timeouts += 1 if result.timeout?
             @rasp.duration_ns += result.duration_ns
             @rasp.duration_ext_ns += result.duration_ext_ns

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -8,7 +8,7 @@ module Datadog
         module_function
 
         def report_rasp(type, result)
-          return if result.is_a?(SecurityEngine::Result::Error)
+          return if result.error?
 
           tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF request metrics via Telemetry.
+      module TelemetryExporter
+        module_function
+
+        def export_waf_request_metrics(metrics, context)
+          AppSec.telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            tags: {
+              waf_version: WAF::VERSION::BASE_STRING,
+              event_rules_version: context.waf_runner_ruleset_version,
+              rule_triggered: metrics.matches.positive?.to_s,
+              waf_error: metrics.errors.positive?.to_s,
+              waf_timeout: metrics.timeouts.positive?.to_s,
+              request_blocked: context.interrupted?.to_s,
+              block_failure: 'false',
+              rate_limited: (!context.trace.sampled?).to_s
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics.rb

@@ -11,3 +11,4 @@ end
 require_relative 'metrics/collector'
 require_relative 'metrics/exporter'
 require_relative 'metrics/telemetry'
+require_relative 'metrics/telemetry_exporter'

data/lib/datadog/appsec/security_engine/engine.rb

@@ -20,8 +20,6 @@ module Datadog
           exclusion_data
         ].freeze
 
-        attr_reader :waf_addresses, :ruleset_version
-
         def initialize(appsec_settings:, telemetry:)
           @default_ruleset = appsec_settings.ruleset
 
@@ -39,9 +37,9 @@ module Datadog
 
           diagnostics = load_default_config(telemetry: telemetry)
           report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
+          @ruleset_version = diagnostics['ruleset_version']
 
-          @waf_handle = @waf_builder.build_handle
-          @waf_addresses = @waf_handle.known_addresses
+          @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
         rescue WAF::Error => e
           error_message = "AppSec security engine failed to initialize"
 
@@ -51,16 +49,8 @@ module Datadog
           raise e
         end
 
-        def finalize!
-          @waf_handle&.finalize!
-          @waf_builder&.finalize!
-
-          @waf_addresses = []
-          @ruleset_version = nil
-        end
-
         def new_runner
-          SecurityEngine::Runner.new(@waf_handle.build_context)
+          SecurityEngine::Runner.new(@handle_ref, ruleset_version: @ruleset_version)
         end
 
         def add_or_update_config(config, path:)
@@ -70,7 +60,7 @@ module Datadog
           remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
 
           diagnostics = @waf_builder.add_or_update_config(config, path: path)
-          @ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
+          @reconfigured_ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
           report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
 
           # we need to load default config if diagnostics contains top-level error for rules or processors
@@ -79,6 +69,7 @@ module Datadog
               diagnostics.dig('rules', 'error') ||
               diagnostics.dig('processors', 'errors'))
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
             report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
           end
 
@@ -95,6 +86,7 @@ module Datadog
 
           if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
             report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
           end
 
@@ -107,24 +99,17 @@ module Datadog
         end
 
         def reconfigure!
-          old_waf_handle = @waf_handle
-
-          @waf_handle = @waf_builder.build_handle
-          @waf_addresses = @waf_handle.known_addresses
+          new_waf_handle = @waf_builder.build_handle
+          @ruleset_version = @reconfigured_ruleset_version
 
-          old_waf_handle&.finalize!
+          @handle_ref.current = new_waf_handle
         rescue WAF::Error => e
-          error_message = "AppSec security engine failed to reconfigure"
+          # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
+          # This means that the current WAF handle was not yet substituted.
+          error_message = "AppSec security engine failed to reconfigure, reverting to the previous configuration"
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           AppSec.telemetry.report(e, description: error_message)
-
-          if old_waf_handle
-            Datadog.logger.warn("Reverting to the previous configuration")
-
-            @waf_handle = old_waf_handle
-            @waf_addresses = old_waf_handle.known_addresses
-          end
         end
 
         private
@@ -141,10 +126,7 @@ module Datadog
           # deprecated - ip passlist should be configured via RC
           config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
 
-          diagnostics = @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
-          @ruleset_version = diagnostics['ruleset_version']
-
-          diagnostics
+          @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
         end
 
         def report_configuration_diagnostics(diagnostics, action:, telemetry:)
@@ -152,7 +134,7 @@ module Datadog
 
           common_tags = {
             waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
-            event_rules_version: diagnostics.fetch('ruleset_version', @ruleset_version).to_s,
+            event_rules_version: diagnostics['ruleset_version'].to_s,
             action: action
           }
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -26,6 +26,10 @@ module Datadog
           def match?
             raise NotImplementedError
           end
+
+          def error?
+            raise NotImplementedError
+          end
         end
 
         # A result that indicates a security rule match
@@ -33,6 +37,10 @@ module Datadog
           def match?
             true
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates a successful security rules check without a match
@@ -40,6 +48,10 @@ module Datadog
           def match?
             false
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates an internal security library error
@@ -60,6 +72,10 @@ module Datadog
           def match?
             false
           end
+
+          def error?
+            true
+          end
         end
       end
     end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -9,9 +9,13 @@ module Datadog
       class Runner
         SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
 
-        def initialize(waf_context)
+        attr_reader :ruleset_version
+
+        def initialize(handle_ref, ruleset_version:)
           @mutex = Mutex.new
-          @waf_context = waf_context
+          @handle_ref = handle_ref
+          @waf_handle = handle_ref.acquire
+          @ruleset_version = ruleset_version
 
           @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
         end
@@ -54,14 +58,24 @@ module Datadog
           @mutex.unlock
         end
 
+        def waf_context
+          @waf_context ||= @waf_handle.build_context
+        end
+
+        def waf_addresses
+          @waf_handle.known_addresses
+        end
+
         def finalize!
-          @waf_context.finalize!
+          @waf_context&.finalize!
+        ensure
+          @handle_ref.release(@waf_handle)
         end
 
         private
 
         def try_run(persistent_data, ephemeral_data, timeout)
-          @waf_context.run(persistent_data, ephemeral_data, timeout)
+          waf_context.run(persistent_data, ephemeral_data, timeout)
         rescue WAF::LibDDWAFError => e
           Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
           AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')

data/lib/datadog/appsec/thread_safe_ref.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used for referencing an object that might be marked
+    # for finalization in another thread.
+    #
+    # References to the object are counted, and objects marked for finalization
+    # can be safely finalized when their reference count reaches zero.
+    class ThreadSafeRef
+      def initialize(initial_obj, finalizer: :finalize!)
+        @current = initial_obj
+        @finalizer = finalizer
+
+        @counters = Hash.new(0)
+        @outdated = []
+        @mutex = Mutex.new
+      end
+
+      def acquire
+        @mutex.synchronize do
+          @counters[@current] += 1
+
+          @current
+        end
+      end
+
+      def release(obj)
+        @mutex.synchronize do
+          @counters[obj] -= 1
+
+          @outdated.reject! do |outdated_obj|
+            next unless @counters[outdated_obj].zero?
+
+            finalize(outdated_obj)
+          end
+        end
+      end
+
+      def current=(obj)
+        @mutex.synchronize do
+          @outdated << @current
+
+          @current = obj
+        end
+      end
+
+      private
+
+      def finalize(obj)
+        obj.public_send(@finalizer)
+
+        true
+      rescue => e
+        Datadog.logger.debug("Couldn't finalize #{obj.class.name} object, error: #{e.inspect}")
+
+        true
+      end
+    end
+  end
+end

data/lib/datadog/appsec/trace_keeper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used to mark trace as manual keep and tag it as ASM product.
+    module TraceKeeper
+      def self.keep!(trace)
+        return unless trace
+
+        # NOTE: This action will not set correct decision maker value, so the
+        #       trace keeping must be done with additional steps below
+        trace.keep!
+
+        # Propagate to downstream services the information that
+        # the current distributed trace is containing at least one ASM event.
+        trace.set_tag(
+          Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+          Tracing::Sampling::Ext::Decision::ASM
+        )
+        trace.set_distributed_source(Ext::PRODUCT_BIT)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/hash_coercion.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      # A module for coercing arbitrary objects into hashes.
+      module HashCoercion
+        # A best effort to coerce an object to a hash with methods known to various
+        # frameworks with a fallback to standard library.
+        #
+        # @param object [Object] The object to coerce.
+        # @return [Hash, nil] The coerced `Hash` or `nil` if the object is not coercible.
+        def self.coerce(object)
+          return object.as_json if object.respond_to?(:as_json)
+          return object.to_hash if object.respond_to?(:to_hash)
+          return object.to_h if object.respond_to?(:to_h)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -34,13 +34,6 @@ module Datadog
         components.appsec&.reconfigure!
       end
 
-      def reconfigure_lock(&block)
-        appsec_component = components.appsec
-        return unless appsec_component
-
-        appsec_component.reconfigure_lock(&block)
-      end
-
       def perform_api_security_check?
         Datadog.configuration.appsec.api_security.enabled &&
           Datadog.configuration.appsec.api_security.sample_rate.sample?

data/lib/datadog/auto_instrument_base.rb

@@ -3,6 +3,7 @@
 module Datadog
   # base methods stubbed for adding auto instrument extensions
   module AutoInstrumentBase
-    def add_auto_instrument; end
+    def add_auto_instrument
+    end
   end
 end