cw-datadog 2.23.0.2

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:, telemetry:)
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
+                'The `:recommended` ruleset will be used instead.' \
+                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
+
+            telemetry.report(e, description: 'libddwaf ruleset failed to load')
+
+            raise e
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
+            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
+
+            data
+          end
+
+          def load_exclusions(ip_passlist: [])
+            return [] if ip_passlist.empty?
+
+            passlist_exclusions(ip_passlist)
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
+            }
+          end
+
+          def passlist_exclusions(ip_passlist) # rubocop:disable Metrics/MethodLength
+            case ip_passlist
+            when Array
+              pass = ip_passlist
+              monitor = []
+            when Hash
+              pass = ip_passlist[:pass]
+              monitor = ip_passlist[:monitor]
+            end
+
+            exclusions = []
+
+            exclusions << {
+              'conditions' => [
+                {
+                  'operator' => 'ip_match',
+                  'parameters' => {
+                    'inputs' => [
+                      {
+                        'address' => 'http.client_ip'
+                      }
+                    ],
+                    'list' => pass
+                  }
+                }
+              ],
+              'id' => SecureRandom.uuid,
+            }
+
+            exclusions << {
+              'conditions' => [
+                {
+                  'operator' => 'ip_match',
+                  'parameters' => {
+                    'inputs' => [
+                      {
+                        'address' => 'http.client_ip'
+                      }
+                    ],
+                    'list' => monitor
+                  }
+                }
+              ],
+              'id' => SecureRandom.uuid,
+              'on_match' => 'monitor'
+            }
+
+            exclusions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../core/rate_limiter'
+
+module Datadog
+  module AppSec
+    # Per-thread rate limiter based on token bucket rate limiter.
+    #
+    # Since AppSec marks sampling to keep on a security event, this limits
+    # the flood of egress traces involving AppSec
+    class RateLimiter
+      THREAD_KEY = :datadog_security_appsec_rate_limiter
+
+      class << self
+        def thread_local
+          rate_limiter = Thread.current.thread_variable_get(THREAD_KEY)
+          return rate_limiter unless rate_limiter.nil?
+
+          Thread.current.thread_variable_set(THREAD_KEY, new(trace_rate_limit))
+        end
+
+        # reset a rate limiter: used for testing
+        def reset!
+          Thread.current.thread_variable_set(THREAD_KEY, nil)
+        end
+
+        private
+
+        def trace_rate_limit
+          Datadog.configuration.appsec.trace_rate_limit
+        end
+      end
+
+      def initialize(rate)
+        @rate_limiter = Core::TokenBucket.new(rate)
+      end
+
+      def limit
+        return yield if @rate_limiter.allow?
+
+        Datadog.logger.debug { "Rate limit hit: #{@rate_limiter.current_window_rate} AppSec traces/second" }
+      end
+    end
+  end
+end

data/lib/datadog/appsec/remote.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative '../core/remote/dispatcher'
+require_relative 'processor/rule_loader'
+
+module Datadog
+  module AppSec
+    # Remote
+    module Remote
+      class ReadError < StandardError; end
+
+      class NoRulesError < StandardError; end
+
+      class << self
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
+
+        # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
+        ASM_CAPABILITIES = [
+          CAP_ASM_IP_BLOCKING,
+          CAP_ASM_USER_BLOCKING,
+          CAP_ASM_EXCLUSIONS,
+          CAP_ASM_REQUEST_BLOCKING,
+          CAP_ASM_RESPONSE_BLOCKING,
+          CAP_ASM_DD_RULES,
+          CAP_ASM_CUSTOM_RULES,
+          CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
+          CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
+        ].freeze
+
+        ASM_PRODUCTS = [
+          'ASM_DD',       # Datadog employee issued configuration
+          'ASM',          # customer issued configuration (rulesets, passlist...)
+          'ASM_FEATURES', # capabilities
+          'ASM_DATA',     # config files (IP addresses or users for blocking)
+        ].freeze
+
+        def capabilities
+          remote_features_enabled? ? ASM_CAPABILITIES : []
+        end
+
+        def products
+          remote_features_enabled? ? ASM_PRODUCTS : []
+        end
+
+        def receivers(telemetry)
+          return [] unless remote_features_enabled?
+
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
+          receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
+            next unless AppSec.security_engine
+
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content || change.type == :delete
+
+              case change.type
+              when :insert, :update
+                AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
+
+                content.applied # steep:ignore
+              when :delete
+                AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
+              end
+            end
+
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
+          end
+
+          [receiver]
+        end
+
+        private
+
+        def remote_features_enabled?
+          Datadog.configuration.appsec.using_default?(:ruleset)
+        end
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require_relative 'assets'
+require_relative 'utils/http/media_range'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      class << self
+        def from_interrupt_params(interrupt_params, http_accept_header)
+          return redirect_response(interrupt_params) if interrupt_params['location']
+
+          block_response(interrupt_params, http_accept_header)
+        end
+
+        private
+
+        def block_response(interrupt_params, http_accept_header)
+          content_type = case interrupt_params['type']
+          when nil, 'auto' then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          end
+
+          Response.new(
+            status: interrupt_params['status_code']&.to_i || 403,
+            headers: {'Content-Type' => content_type},
+            body: [content(content_type)],
+          )
+        end
+
+        def redirect_response(interrupt_params)
+          status_code = interrupt_params['status_code'].to_i
+
+          Response.new(
+            status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
+            headers: {'Location' => interrupt_params.fetch('location')},
+            body: [],
+          )
+        end
+
+        CONTENT_TYPE_TO_FORMAT = {
+          'application/json' => :json,
+          'text/html' => :html,
+          'text/plain' => :text,
+        }.freeze
+
+        FORMAT_TO_CONTENT_TYPE = {
+          'json' => 'application/json',
+          'html' => 'text/html',
+        }.freeze
+
+        DEFAULT_CONTENT_TYPE = 'application/json'
+
+        def content_type(http_accept_header)
+          return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
+
+          accept_types = http_accept_header.split(',').map(&:strip)
+
+          accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
+
+          accepted.each do |range|
+            type_match = CONTENT_TYPE_TO_FORMAT.keys.find { |type| range === type }
+
+            return type_match if type_match
+          end
+
+          DEFAULT_CONTENT_TYPE
+        rescue Datadog::AppSec::Utils::HTTP::MediaRange::ParseError
+          DEFAULT_CONTENT_TYPE
+        end
+
+        def content(content_type)
+          content_format = CONTENT_TYPE_TO_FORMAT[content_type]
+
+          using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
+
+          if using_default
+            Datadog::AppSec::Assets.blocked(format: content_format)
+          else
+            Datadog.configuration.appsec.block.templates.send(content_format)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/sample_rate.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # SampleRate basic sample rate
+    class SampleRate
+      attr_reader :rate
+
+      def initialize(rate)
+        @rate = rate
+      end
+
+      def sample?
+        return false if rate <= 0
+        return true if rate >= 1
+
+        Kernel.rand < rate
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/engine.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # SecurityEngine::Engine creates WAF builder and manages its configuration.
+      # It also rebuilds WAF handle from the WAF builder when configuration changes.
+      class Engine
+        DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
+        TELEMETRY_ACTIONS = %w[init update].freeze
+        DIAGNOSTICS_CONFIG_KEYS = %w[
+          rules
+          custom_rules
+          exclusions
+          actions
+          processors
+          scanners
+          rules_override
+          rules_data
+          exclusion_data
+        ].freeze
+
+        def initialize(appsec_settings:, telemetry:)
+          @default_ruleset = appsec_settings.ruleset
+
+          # NOTE: replace appsec_settings argument with default_ruleset when removing these deprecated settings
+          @default_ip_denylist = appsec_settings.ip_denylist
+          @default_user_id_denylist = appsec_settings.user_id_denylist
+          @default_ip_passlist = appsec_settings.ip_passlist
+
+          @waf_builder = WAF::HandleBuilder.new(
+            obfuscator: {
+              key_regex: appsec_settings.obfuscator_key_regex,
+              value_regex: appsec_settings.obfuscator_value_regex
+            }
+          )
+
+          diagnostics = load_default_config(telemetry: telemetry)
+          report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
+          @ruleset_version = diagnostics['ruleset_version']
+
+          @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
+        rescue WAF::Error => e
+          error_message = "AppSec security engine failed to initialize"
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          telemetry.report(e, description: error_message)
+
+          raise e
+        end
+
+        def new_runner
+          SecurityEngine::Runner.new(@handle_ref, ruleset_version: @ruleset_version)
+        end
+
+        def add_or_update_config(config, path:)
+          @is_ruleset_update = path.include?('ASM_DD')
+
+          # default config has to be removed when adding an ASM_DD config
+          remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
+
+          diagnostics = @waf_builder.add_or_update_config(config, path: path)
+          @reconfigured_ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
+          report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+
+          # we need to load default config if diagnostics contains top-level error for rules or processors
+          if @is_ruleset_update &&
+              (diagnostics.key?('error') ||
+              diagnostics.dig('rules', 'error') ||
+              diagnostics.dig('processors', 'errors'))
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          diagnostics
+        rescue WAF::Error => e
+          error_message = "libddwaf builder failed to add or update config at path: #{path}"
+
+          Datadog.logger.debug("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def remove_config_at_path(path)
+          result = @waf_builder.remove_config_at_path(path)
+
+          if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
+            diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
+            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          end
+
+          result
+        rescue WAF::Error => e
+          error_message = "libddwaf handle builder failed to remove config at path: #{path}"
+
+          Datadog.logger.error("#{error_message}, error: #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        def reconfigure!
+          new_waf_handle = @waf_builder.build_handle
+          @ruleset_version = @reconfigured_ruleset_version
+
+          @handle_ref.current = new_waf_handle
+        rescue WAF::Error => e
+          # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
+          # This means that the current WAF handle was not yet substituted.
+          error_message = "AppSec security engine failed to reconfigure, reverting to the previous configuration"
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          AppSec.telemetry.report(e, description: error_message)
+        end
+
+        private
+
+        def load_default_config(telemetry:)
+          config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
+
+          # deprecated - ip and user id denylists should be configured via RC
+          config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: @default_ip_denylist,
+            user_id_denylist: @default_user_id_denylist
+          )
+
+          # deprecated - ip passlist should be configured via RC
+          config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
+
+          @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
+        end
+
+        def report_configuration_diagnostics(diagnostics, action:, telemetry:)
+          raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
+
+          common_tags = {
+            waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+            event_rules_version: diagnostics['ruleset_version'].to_s,
+            action: action
+          }
+
+          if diagnostics['error']
+            telemetry.inc(
+              Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+              tags: common_tags.merge(scope: 'top-level')
+            )
+
+            telemetry.error(diagnostics['error'])
+          end
+
+          diagnostics.each do |config_key, config_diagnostics|
+            next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
+            next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
+
+            if config_diagnostics['error']
+              telemetry.error(config_diagnostics['error'])
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
+                tags: common_tags.merge(config_key: config_key, scope: 'top-level')
+              )
+            elsif config_diagnostics['errors']
+              config_diagnostics['errors'].each do |error, config_ids|
+                telemetry.error("#{error}: [#{config_ids.join(",")}]")
+              end
+
+              telemetry.inc(
+                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
+                tags: common_tags.merge(config_key: config_key, scope: 'item')
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, attributes:, duration_ns:, duration_ext_ns:, timeout:, keep:, input_truncated:)
+            @events = events
+            @actions = actions
+            @attributes = attributes
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+
+            @keep = !!keep
+            @timeout = !!timeout
+            @input_truncated = !!input_truncated
+          end
+
+          def timeout?
+            @timeout
+          end
+
+          def keep?
+            @keep
+          end
+
+          def input_truncated?
+            @input_truncated
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+
+          def error?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+
+          def error?
+            false
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+
+          def error?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:, input_truncated:)
+            @events = []
+            @actions = @attributes = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+            @input_truncated = !!input_truncated
+          end
+
+          def keep?
+            false
+          end
+
+          def timeout?
+            false
+          end
+
+          def input_truncated?
+            @input_truncated
+          end
+
+          def match?
+            false
+          end
+
+          def error?
+            true
+          end
+        end
+      end
+    end
+  end
+end