datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/security_engine/engine.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # SecurityEngine::Engine creates WAF builder and manages its configuration.
+      # It also rebuilds WAF handle from the WAF builder when configuration changes.
+      class Engine
+        DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
+        TELEMETRY_ACTIONS = %w[init update].freeze
+        DIAGNOSTICS_CONFIG_KEYS = %w[
+          rules
+          custom_rules
+          exclusions
+          actions
+          processors
+          scanners
+          rules_override
+          rules_data
+          exclusion_data
+        ].freeze
+
+        def initialize(appsec_settings:, telemetry:)
+          @default_ruleset = appsec_settings.ruleset
+
+          # NOTE: replace appsec_settings argument with default_ruleset when removing these deprecated settings
+          @default_ip_denylist = appsec_settings.ip_denylist
+          @default_user_id_denylist = appsec_settings.user_id_denylist
+          @default_ip_passlist = appsec_settings.ip_passlist
+
+          @waf_builder = WAF::HandleBuilder.new(
+            obfuscator: {
+              key_regex: appsec_settings.obfuscator_key_regex,
+              value_regex: appsec_settings.obfuscator_value_regex
+            }
+          )
+
+          diagnostics = load_default_config(telemetry: telemetry)
+          report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
+          @ruleset_version = diagnostics['ruleset_version']
+
+          @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
+
+          metric('init', success: true, ruleset_version: @ruleset_version, telemetry: telemetry)
+        rescue WAF::Error => e
+          error_message = 'AppSec security engine failed to initialize'
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          telemetry.report(e, description: error_message)
+
+          metric('init', success: false, ruleset_version: @ruleset_version, telemetry: telemetry)
+
+          raise e
+        end
+
+        def new_runner
+          SecurityEngine::Runner.new(@handle_ref, ruleset_version: @ruleset_version)
+        end
+
+        def add_or_update_config(config, path:)
+          is_ruleset_update = path.include?('ASM_DD')
+
+          # default config has to be removed when adding an ASM_DD config
+          remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if is_ruleset_update
+
+          diagnostics = @waf_builder.add_or_update_config(config, path: path)
+          @reconfigured_ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
+          report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+
+          # we need to load default config if diagnostics contains top-level error for rules or processors
+          if is_ruleset_update &&
+              (diagnostics.key?('error') ||
+              diagnostics.dig('rules', 'error') ||
+              diagnostics.dig('processors', 'errors'))
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          diagnostics
+        rescue WAF::Error => e
+          error_message = "libddwaf builder failed to add or update config at path: #{path}"
+
+          Datadog.logger.debug("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def remove_config_at_path(path)
+          result = @waf_builder.remove_config_at_path(path)
+
+          if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          result
+        rescue WAF::Error => e
+          error_message = "libddwaf handle builder failed to remove config at path: #{path}"
+
+          Datadog.logger.error("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def reconfigure!
+          new_waf_handle = @waf_builder.build_handle
+          @ruleset_version = @reconfigured_ruleset_version
+
+          @handle_ref.current = new_waf_handle
+
+          metric('updates', success: true, ruleset_version: @ruleset_version, telemetry: AppSec.telemetry)
+        rescue WAF::Error => e
+          # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
+          # This means that the current WAF handle was not yet substituted.
+          error_message = 'AppSec security engine failed to reconfigure, reverting to the previous configuration'
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+
+          metric('updates', success: false, ruleset_version: @reconfigured_ruleset_version, telemetry: AppSec.telemetry)
+        end
+
+        private
+
+        def metric(metric_name, success:, ruleset_version:, telemetry:)
+          telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE,
+            "waf.#{metric_name}",
+            1,
+            tags: {
+              waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+              event_rules_version: ruleset_version.to_s,
+              success: success.to_s
+            }
+          )
+        end
+
+        def load_default_config(telemetry:)
+          config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
+
+          # deprecated - ip and user id denylists should be configured via RC
+          config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: @default_ip_denylist,
+            user_id_denylist: @default_user_id_denylist
+          )
+
+          # deprecated - ip passlist should be configured via RC
+          config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
+
+          @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
+        end
+
+        def report_configuration_diagnostics(diagnostics, action:, telemetry:)
+          raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
+
+          common_tags = {
+            waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+            event_rules_version: diagnostics['ruleset_version'].to_s,
+            action: action
+          }
+
+          if diagnostics['error']
+            telemetry.inc(
+              Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+              tags: common_tags.merge(scope: 'top-level')
+            )
+
+            telemetry.error(diagnostics['error'])
+          end
+
+          diagnostics.each do |config_key, config_diagnostics|
+            next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
+            next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
+
+            if config_diagnostics['error']
+              telemetry.error(config_diagnostics['error'])
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+                tags: common_tags.merge(config_key: config_key, scope: 'top-level')
+              )
+            elsif config_diagnostics['errors']
+              config_diagnostics['errors'].each do |error, config_ids|
+                telemetry.error("#{error}: [#{config_ids.join(",")}]")
+              end
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
+                tags: common_tags.merge(config_key: config_key, scope: 'item')
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/result.rb

@@ -7,25 +7,39 @@ module Datadog
       module Result
         # A generic result without indication of its type.
         class Base
-          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
 
-          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+          def initialize(events:, actions:, attributes:, duration_ns:, duration_ext_ns:, timeout:, keep:, input_truncated:)
             @events = events
             @actions = actions
-            @derivatives = derivatives
-
-            @timeout = timeout
+            @attributes = attributes
             @duration_ns = duration_ns
             @duration_ext_ns = duration_ext_ns
+
+            @keep = !!keep
+            @timeout = !!timeout
+            @input_truncated = !!input_truncated
           end
 
           def timeout?
-            !!@timeout
+            @timeout
+          end
+
+          def keep?
+            @keep
+          end
+
+          def input_truncated?
+            @input_truncated
           end
 
           def match?
             raise NotImplementedError
           end
+
+          def error?
+            raise NotImplementedError
+          end
         end
 
         # A result that indicates a security rule match
@@ -33,6 +47,10 @@ module Datadog
           def match?
             true
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates a successful security rules check without a match
@@ -40,26 +58,44 @@ module Datadog
           def match?
             false
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates an internal security library error
         class Error
-          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
 
-          def initialize(duration_ext_ns:)
+          def initialize(duration_ext_ns:, input_truncated:)
             @events = []
-            @actions = @derivatives = {}
+            @actions = {}.freeze
+            @attributes = {}.freeze
             @duration_ns = 0
             @duration_ext_ns = duration_ext_ns
+            @input_truncated = !!input_truncated
+          end
+
+          def keep?
+            false
           end
 
           def timeout?
             false
           end
 
+          def input_truncated?
+            @input_truncated
+          end
+
           def match?
             false
           end
+
+          def error?
+            true
+          end
         end
       end
     end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -9,10 +9,13 @@ module Datadog
       class Runner
         SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
 
-        def initialize(handle, telemetry:)
+        attr_reader :ruleset_version
+
+        def initialize(handle_ref, ruleset_version:)
           @mutex = Mutex.new
-          @context = WAF::Context.new(handle)
-          @telemetry = telemetry
+          @handle_ref = handle_ref
+          @waf_handle = handle_ref.acquire
+          @ruleset_version = ruleset_version
 
           @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
         end
@@ -24,54 +27,74 @@ module Datadog
           persistent_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? || v.empty?
+            v.nil? || (v.respond_to?(:empty?) && v.empty?)
           end
 
           ephemeral_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? || v.empty?
+            v.nil? || (v.respond_to?(:empty?) && v.empty?)
           end
 
-          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          result = try_run(persistent_data, ephemeral_data, timeout)
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
 
           report_execution(result)
 
           unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
-            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns, input_truncated: result.input_truncated?)
           end
 
           klass = (result.status == :match) ? Result::Match : Result::Ok
           klass.new(
             events: result.events,
             actions: result.actions,
-            derivatives: result.derivatives,
-            timeout: result.timeout,
-            duration_ns: result.total_runtime,
-            duration_ext_ns: (stop_ns - start_ns)
+            attributes: result.attributes,
+            keep: result.keep?,
+            timeout: result.timeout?,
+            duration_ns: result.duration,
+            duration_ext_ns: (stop_ns - start_ns),
+            input_truncated: result.input_truncated?
           )
         ensure
           @mutex.unlock
         end
 
-        def finalize
-          @context.finalize
+        def waf_context
+          @waf_context ||= @waf_handle.build_context
+        end
+
+        def waf_addresses
+          @waf_handle.known_addresses
+        end
+
+        def finalize!
+          @waf_context&.finalize!
+        ensure
+          @handle_ref.release(@waf_handle)
         end
 
         private
 
         def try_run(persistent_data, ephemeral_data, timeout)
-          @context.run(persistent_data, ephemeral_data, timeout)
-        rescue WAF::LibDDWAF::Error => e
-          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
-          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
-
-          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+          waf_context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAFError => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e.class}: #{e.message} backtrace: #{e.backtrace&.first(3)}" }
+          AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          WAF::Result.new(
+            status: :err_internal,
+            events: [],
+            actions: {},
+            attributes: {},
+            duration: 0,
+            keep: false,
+            timeout: false
+          )
         end
 
         def report_execution(result)
-          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout?
 
           if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
             Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
@@ -79,7 +102,7 @@ module Datadog
             message = "#{@debug_tag} execution error: #{result.status.inspect}"
 
             Datadog.logger.debug { message }
-            @telemetry.error(message)
+            AppSec.telemetry.error(message)
           end
         end
       end

data/lib/datadog/appsec/security_event.rb

@@ -3,7 +3,7 @@
 module Datadog
   module AppSec
     # A class that represents a security event of any kind. It could be an event
-    # representing an attack or fingerprinting results as derivatives or an API
+    # representing an attack or fingerprinting results as attributes or an API
     # security check with extracted schema.
     class SecurityEvent
       SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
@@ -17,22 +17,20 @@ module Datadog
         @span = span
       end
 
-      def attack?
-        return @is_attack if defined?(@is_attack)
-
-        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      def keep?
+        @waf_result.keep?
       end
 
       def schema?
         return @has_schema if defined?(@has_schema)
 
-        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+        @has_schema = @waf_result.attributes.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
       end
 
       def fingerprint?
         return @has_fingerprint if defined?(@has_fingerprint)
 
-        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+        @has_fingerprint = @waf_result.attributes.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
       end
     end
   end

data/lib/datadog/appsec/thread_safe_ref.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used for referencing an object that might be marked
+    # for finalization in another thread.
+    #
+    # References to the object are counted, and objects marked for finalization
+    # can be safely finalized when their reference count reaches zero.
+    class ThreadSafeRef
+      def initialize(initial_obj, finalizer: :finalize!)
+        @current = initial_obj
+        @finalizer = finalizer
+
+        @counters = Hash.new(0)
+        @outdated = []
+        @mutex = Mutex.new
+      end
+
+      def acquire
+        @mutex.synchronize do
+          @counters[@current] += 1
+
+          @current
+        end
+      end
+
+      def release(obj)
+        @mutex.synchronize do
+          @counters[obj] -= 1
+
+          @outdated.reject! do |outdated_obj|
+            next unless @counters[outdated_obj].zero?
+
+            finalize(outdated_obj)
+          end
+        end
+      end
+
+      def current=(obj)
+        @mutex.synchronize do
+          @outdated << @current
+
+          @current = obj
+        end
+      end
+
+      private
+
+      def finalize(obj)
+        obj.public_send(@finalizer)
+
+        true
+      rescue => e
+        Datadog.logger.debug("Couldn't finalize #{obj.class.name} object, error: #{e.inspect}")
+
+        true
+      end
+    end
+  end
+end

data/lib/datadog/appsec/trace_keeper.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used to mark trace as manual keep and tag it as ASM product.
+    module TraceKeeper
+      def self.keep!(trace)
+        return unless trace
+
+        previous_dm = trace.get_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+
+        # NOTE: This action will not set correct decision maker value, so the
+        #       trace keeping must be done with additional steps below
+        trace.keep!
+
+        # NOTE: Preserve decision maker if already set by another product.
+        #       As `trace.keep!` resets `_dd.p.dm` to `MANUAL`, we restore the
+        #       previous value when another product has already claimed the
+        #       decision maker.
+        if previous_dm.nil? || previous_dm == Tracing::Sampling::Ext::Decision::MANUAL
+          trace.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            Tracing::Sampling::Ext::Decision::ASM
+          )
+        else
+          trace.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            previous_dm,
+          )
+        end
+
+        trace.set_distributed_source(Ext::PRODUCT_BIT)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/hash_coercion.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      # A module for coercing arbitrary objects into hashes.
+      module HashCoercion
+        # A best effort to coerce an object to a hash with methods known to various
+        # frameworks with a fallback to standard library.
+        #
+        # @param object [Object] The object to coerce.
+        # @return [Hash, nil] The coerced `Hash` or `nil` if the object is not coercible.
+        def self.coerce(object)
+          return object.as_json if object.respond_to?(:as_json)
+          return object.to_hash if object.respond_to?(:to_hash)
+          return object.to_h if object.respond_to?(:to_h)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/body.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'cgi'
+
+require_relative 'url_encoded'
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for handling HTTP body parsing
+        module Body
+          def self.parse(body, media_type:)
+            return if body.nil?
+
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+            # @type var content: ::String?
+            content = body.respond_to?(:read) ? body.read : body # steep:ignore NoMethod, IncompatibleAssignment
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+
+            return if content.nil? || content.empty?
+
+            if media_type.subtype == 'json' || media_type.subtype.end_with?('+json')
+              JSON.parse(content)
+            elsif media_type.subtype == 'x-www-form-urlencoded'
+              URLEncoded.parse(content)
+            end
+          rescue => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to parse body')
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/body_reader.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        module BodyReader
+          module_function
+
+          def read(body, limit:, rewind_before_read: false)
+            return body.byteslice(0, limit + 1) if body.is_a?(String)
+            return if body.nil? || !body.respond_to?(:read)
+            return if rewind_before_read && !body.respond_to?(:rewind)
+
+            read_stream(body, limit: limit, rewind_before_read: rewind_before_read)
+          end
+
+          def read_stream(body, limit:, rewind_before_read: false)
+            # @type var rewound: bool
+            rewound = false
+
+            if rewind_before_read
+              return unless rewind(body)
+
+              rewound = true
+            end
+
+            buffer = +''.b
+            max = limit + 1
+
+            while buffer.bytesize <= limit
+              chunk = body.read(max - buffer.bytesize)
+              break if chunk.nil? || chunk.empty?
+
+              buffer << chunk
+            end
+
+            buffer
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to read body: #{e.class}: #{e.message}" }
+
+            raise
+          ensure
+            rewind(body) if rewound
+          end
+
+          private_class_method def rewind(body)
+            return false unless body.respond_to?(:rewind)
+
+            # NOTE: Steep can't narrow interfaces via `respond_to?` guard-clause
+            body.rewind # steep:ignore
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind body: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_range.rb

@@ -168,7 +168,8 @@ module Datadog
           #
           # returns true if the MediaType is accepted by this MediaRange
           def ===(other)
-            return self === MediaType.new(other) if other.is_a?(::String)
+            return false if other.nil?
+            return self === MediaType.parse(other) if other.is_a?(::String)
 
             type == other.type && subtype == other.subtype && other.parameters.all? { |k, v| parameters[k] == v } ||
               type == other.type && wildcard?(:subtype) ||