datadog 2.12.1 → 2.19.0

data/lib/datadog/appsec/remote.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_merger'
 require_relative 'processor/rule_loader'
 
 module Datadog
@@ -9,20 +8,23 @@ module Datadog
     # Remote
     module Remote
       class ReadError < StandardError; end
+
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
+        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
+        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
+        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
+        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
+        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
+        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
+        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
+        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
+        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,6 +37,8 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -52,67 +56,35 @@ module Datadog
           remote_features_enabled? ? ASM_PRODUCTS : []
         end
 
-        # rubocop:disable Metrics/MethodLength
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
-            changes.each do |change|
-              Datadog.logger.debug { "remote config change: '#{change.path}'" }
-            end
-
-            rules = []
-            custom_rules = []
-            data = []
-            overrides = []
-            exclusions = []
-
-            repository.contents.each do |content|
-              parsed_content = parse_content(content)
-
-              case content.path.product
-              when 'ASM_DD'
-                rules << parsed_content
-              when 'ASM_DATA'
-                data << parsed_content['rules_data'] if parsed_content['rules_data']
-              when 'ASM'
-                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
-                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
-                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-              end
-            end
+            next unless AppSec.security_engine
 
-            if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(
-                telemetry: telemetry,
-                ruleset: Datadog.configuration.appsec.ruleset
-              )
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content || change.type == :delete
 
-              raise NoRulesError, 'no default rules available' unless settings_rules
+              case change.type
+              when :insert, :update
+                AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
 
-              rules = [settings_rules]
+                content.applied # steep:ignore
+              when :delete
+                AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
+              end
             end
 
-            ruleset = AppSec::Processor::RuleMerger.merge(
-              rules: rules,
-              data: data,
-              overrides: overrides,
-              exclusions: exclusions,
-              custom_rules: custom_rules,
-              telemetry: telemetry
-            )
-
-            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
-
-            repository.contents.each do |content|
-              content.applied if ASM_PRODUCTS.include?(content.path.product)
-            end
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
           end
 
           [receiver]
         end
-        # rubocop:enable Metrics/MethodLength
 
         private
 

data/lib/datadog/appsec/response.rb

@@ -30,13 +30,13 @@ module Datadog
 
         def block_response(interrupt_params, http_accept_header)
           content_type = case interrupt_params['type']
-                         when nil, 'auto' then content_type(http_accept_header)
-                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
-                         end
+          when nil, 'auto' then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          end
 
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
+            headers: {'Content-Type' => content_type},
             body: [content(content_type)],
           )
         end
@@ -45,8 +45,8 @@ module Datadog
           status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-            headers: { 'Location' => interrupt_params.fetch('location') },
+            status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
+            headers: {'Location' => interrupt_params.fetch('location')},
             body: [],
           )
         end

data/lib/datadog/appsec/security_engine/engine.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # SecurityEngine::Engine creates WAF builder and manages its configuration.
+      # It also rebuilds WAF handle from the WAF builder when configuration changes.
+      class Engine
+        DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
+        TELEMETRY_ACTIONS = %w[init update].freeze
+        DIAGNOSTICS_CONFIG_KEYS = %w[
+          rules
+          custom_rules
+          exclusions
+          actions
+          processors
+          scanners
+          rules_override
+          rules_data
+          exclusion_data
+        ].freeze
+
+        attr_reader :waf_addresses, :ruleset_version
+
+        def initialize(appsec_settings:, telemetry:)
+          @default_ruleset = appsec_settings.ruleset
+
+          # NOTE: replace appsec_settings argument with default_ruleset when removing these deprecated settings
+          @default_ip_denylist = appsec_settings.ip_denylist
+          @default_user_id_denylist = appsec_settings.user_id_denylist
+          @default_ip_passlist = appsec_settings.ip_passlist
+
+          @waf_builder = WAF::HandleBuilder.new(
+            obfuscator: {
+              key_regex: appsec_settings.obfuscator_key_regex,
+              value_regex: appsec_settings.obfuscator_value_regex
+            }
+          )
+
+          diagnostics = load_default_config(telemetry: telemetry)
+          report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
+
+          @waf_handle = @waf_builder.build_handle
+          @waf_addresses = @waf_handle.known_addresses
+        rescue WAF::Error => e
+          error_message = "AppSec security engine failed to initialize"
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          telemetry.report(e, description: error_message)
+
+          raise e
+        end
+
+        def finalize!
+          @waf_handle&.finalize!
+          @waf_builder&.finalize!
+
+          @waf_addresses = []
+          @ruleset_version = nil
+        end
+
+        def new_runner
+          SecurityEngine::Runner.new(@waf_handle.build_context)
+        end
+
+        def add_or_update_config(config, path:)
+          @is_ruleset_update = path.include?('ASM_DD')
+
+          # default config has to be removed when adding an ASM_DD config
+          remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
+
+          diagnostics = @waf_builder.add_or_update_config(config, path: path)
+          @ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
+          report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+
+          # we need to load default config if diagnostics contains top-level error for rules or processors
+          if @is_ruleset_update &&
+              (diagnostics.key?('error') ||
+              diagnostics.dig('rules', 'error') ||
+              diagnostics.dig('processors', 'errors'))
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          diagnostics
+        rescue WAF::Error => e
+          error_message = "libddwaf builder failed to add or update config at path: #{path}"
+
+          Datadog.logger.debug("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def remove_config_at_path(path)
+          result = @waf_builder.remove_config_at_path(path)
+
+          if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          result
+        rescue WAF::Error => e
+          error_message = "libddwaf handle builder failed to remove config at path: #{path}"
+
+          Datadog.logger.error("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def reconfigure!
+          old_waf_handle = @waf_handle
+
+          @waf_handle = @waf_builder.build_handle
+          @waf_addresses = @waf_handle.known_addresses
+
+          old_waf_handle&.finalize!
+        rescue WAF::Error => e
+          error_message = "AppSec security engine failed to reconfigure"
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+
+          if old_waf_handle
+            Datadog.logger.warn("Reverting to the previous configuration")
+
+            @waf_handle = old_waf_handle
+            @waf_addresses = old_waf_handle.known_addresses
+          end
+        end
+
+        private
+
+        def load_default_config(telemetry:)
+          config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
+
+          # deprecated - ip and user id denylists should be configured via RC
+          config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: @default_ip_denylist,
+            user_id_denylist: @default_user_id_denylist
+          )
+
+          # deprecated - ip passlist should be configured via RC
+          config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
+
+          diagnostics = @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
+          @ruleset_version = diagnostics['ruleset_version']
+
+          diagnostics
+        end
+
+        def report_configuration_diagnostics(diagnostics, action:, telemetry:)
+          raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
+
+          common_tags = {
+            waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+            event_rules_version: diagnostics.fetch('ruleset_version', @ruleset_version).to_s,
+            action: action
+          }
+
+          if diagnostics['error']
+            telemetry.inc(
+              Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+              tags: common_tags.merge(scope: 'top-level')
+            )
+
+            telemetry.error(diagnostics['error'])
+          end
+
+          diagnostics.each do |config_key, config_diagnostics|
+            next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
+            next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
+
+            if config_diagnostics['error']
+              telemetry.error(config_diagnostics['error'])
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+                tags: common_tags.merge(config_key: config_key, scope: 'top-level')
+              )
+            elsif config_diagnostics['errors']
+              config_diagnostics['errors'].each do |error, config_ids|
+                telemetry.error("#{error}: [#{config_ids.join(",")}]")
+              end
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
+                tags: common_tags.merge(config_key: config_key, scope: 'item')
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -9,10 +9,9 @@ module Datadog
       class Runner
         SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
 
-        def initialize(handle, telemetry:)
+        def initialize(waf_context)
           @mutex = Mutex.new
-          @context = WAF::Context.new(handle)
-          @telemetry = telemetry
+          @waf_context = waf_context
 
           @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
         end
@@ -24,16 +23,16 @@ module Datadog
           persistent_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
           ephemeral_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
-          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          result = try_run(persistent_data, ephemeral_data, timeout)
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
 
           report_execution(result)
@@ -42,7 +41,7 @@ module Datadog
             return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
           end
 
-          klass = result.status == :match ? Result::Match : Result::Ok
+          klass = (result.status == :match) ? Result::Match : Result::Ok
           klass.new(
             events: result.events,
             actions: result.actions,
@@ -55,19 +54,19 @@ module Datadog
           @mutex.unlock
         end
 
-        def finalize
-          @context.finalize
+        def finalize!
+          @waf_context.finalize!
         end
 
         private
 
         def try_run(persistent_data, ephemeral_data, timeout)
-          @context.run(persistent_data, ephemeral_data, timeout)
-        rescue WAF::LibDDWAF::Error => e
+          @waf_context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAFError => e
           Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
-          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+          AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
 
-          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+          WAF::Result.new(:err_internal, [], 0, false, [], [])
         end
 
         def report_execution(result)
@@ -79,7 +78,7 @@ module Datadog
             message = "#{@debug_tag} execution error: #{result.status.inspect}"
 
             Datadog.logger.debug { message }
-            @telemetry.error(message)
+            AppSec.telemetry.error(message)
           end
         end
       end

data/lib/datadog/appsec/security_event.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A class that represents a security event of any kind. It could be an event
+    # representing an attack or fingerprinting results as derivatives or an API
+    # security check with extracted schema.
+    class SecurityEvent
+      SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      FINGERPRINT_KEY_PREFIX = '_dd.appsec.fp.'
+
+      attr_reader :waf_result, :trace, :span
+
+      def initialize(waf_result, trace:, span:)
+        @waf_result = waf_result
+        @trace = trace
+        @span = span
+      end
+
+      def attack?
+        return @is_attack if defined?(@is_attack)
+
+        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      end
+
+      def schema?
+        return @has_schema if defined?(@has_schema)
+
+        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+      end
+
+      def fingerprint?
+        return @has_fingerprint if defined?(@has_fingerprint)
+
+        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -26,15 +26,12 @@ module Datadog
         components.appsec&.telemetry
       end
 
-      def processor
-        components.appsec&.processor
+      def security_engine
+        components.appsec&.security_engine
       end
 
-      def reconfigure(ruleset:, telemetry:)
-        appsec_component = components.appsec
-        return unless appsec_component
-
-        appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
+      def reconfigure!
+        components.appsec&.reconfigure!
       end
 
       def reconfigure_lock(&block)
@@ -44,7 +41,7 @@ module Datadog
         appsec_component.reconfigure_lock(&block)
       end
 
-      def api_security_enabled?
+      def perform_api_security_check?
         Datadog.configuration.appsec.api_security.enabled &&
           Datadog.configuration.appsec.api_security.sample_rate.sample?
       end

data/lib/datadog/core/buffer/random.rb

@@ -40,7 +40,23 @@ module Datadog
           add_all!(underflow) unless underflow.nil?
 
           # Iteratively replace items, to ensure pseudo-random replacement.
-          overflow.each { |item| replace!(item) } unless overflow.nil?
+          overflow&.each { |item| replace!(item) }
+        end
+
+        def unshift(*items)
+          # TODO The existing concat implementation does not always append
+          # to the end of the buffer - if the buffer is full, a random
+          # item is deleted and the new item is added in the position of
+          # removed item.
+          # Therefore, if we want to preserve the item order, concat
+          # would also need to be changed to maintain order.
+          # With the existing implementation, the idea is to not move
+          # existing items around, which is what sets unshift apart from
+          # concat to begin with.
+          #
+          # Since this method currently delegates to +concat+, it does not
+          # have a matching definition in the thread-safe worker.
+          concat(items)
         end
 
         # Stored items are returned and the local buffer is reset.
@@ -78,7 +94,7 @@ module Datadog
           underflow = nil
           overflow = nil
 
-          overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+          overflow_size = (@max_size > 0) ? (@items.length + items.length) - @max_size : 0
 
           if overflow_size > 0
             # Items will overflow

data/lib/datadog/core/configuration/agent_settings.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module Core
+    module Configuration
+      # Immutable container for the resulting settings
+      class AgentSettings
+        # IPv6 regular expression from
+        # https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
+        # Does not match IPv4 addresses.
+        IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
+
+        attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
+
+        def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
+          @adapter = adapter
+          @ssl = ssl
+          @hostname = hostname
+          @port = port
+          @uds_path = uds_path
+          @timeout_seconds = timeout_seconds
+          freeze
+        end
+
+        def url
+          case adapter
+          when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            hostname = self.hostname
+            hostname = "[#{hostname}]" if IPV6_REGEXP.match?(hostname)
+            "#{ssl ? "https" : "http"}://#{hostname}:#{port}/"
+          when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+            "unix://#{uds_path}"
+          else
+            raise ArgumentError, "Unexpected adapter: #{adapter}"
+          end
+        end
+
+        def ==(other)
+          self.class == other.class &&
+            adapter == other.adapter &&
+            ssl == other.ssl &&
+            hostname == other.hostname &&
+            port == other.port &&
+            uds_path == other.uds_path &&
+            timeout_seconds == other.timeout_seconds
+        end
+      end
+    end
+  end
+end