datadog 2.16.0 → 2.18.0

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.13.3"
+    "rules_version": "1.14.2"
   },
   "rules": [
     {
@@ -1750,24 +1750,7 @@
     {
       "id": "http-endpoint-fingerprint",
       "generator": "http_endpoint_fingerprint",
-      "conditions": [
-        {
-          "operator": "exists",
-          "parameters": {
-            "inputs": [
-              {
-                "address": "waf.context.event"
-              },
-              {
-                "address": "server.business_logic.users.login.failure"
-              },
-              {
-                "address": "server.business_logic.users.login.success"
-              }
-            ]
-          }
-        }
-      ],
+      "conditions": [],
       "parameters": {
         "mappings": [
           {
@@ -1795,7 +1778,7 @@
           }
         ]
       },
-      "evaluate": false,
+      "evaluate": true,
       "output": true
     },
     {
@@ -1951,24 +1934,7 @@
     {
       "id": "http-header-fingerprint",
       "generator": "http_header_fingerprint",
-      "conditions": [
-        {
-          "operator": "exists",
-          "parameters": {
-            "inputs": [
-              {
-                "address": "waf.context.event"
-              },
-              {
-                "address": "server.business_logic.users.login.failure"
-              },
-              {
-                "address": "server.business_logic.users.login.success"
-              }
-            ]
-          }
-        }
-      ],
+      "conditions": [],
       "parameters": {
         "mappings": [
           {
@@ -1981,30 +1947,13 @@
           }
         ]
       },
-      "evaluate": false,
+      "evaluate": true,
       "output": true
     },
     {
       "id": "http-network-fingerprint",
       "generator": "http_network_fingerprint",
-      "conditions": [
-        {
-          "operator": "exists",
-          "parameters": {
-            "inputs": [
-              {
-                "address": "waf.context.event"
-              },
-              {
-                "address": "server.business_logic.users.login.failure"
-              },
-              {
-                "address": "server.business_logic.users.login.success"
-              }
-            ]
-          }
-        }
-      ],
+      "conditions": [],
       "parameters": {
         "mappings": [
           {
@@ -2017,30 +1966,13 @@
           }
         ]
       },
-      "evaluate": false,
+      "evaluate": true,
       "output": true
     },
     {
       "id": "session-fingerprint",
       "generator": "session_fingerprint",
-      "conditions": [
-        {
-          "operator": "exists",
-          "parameters": {
-            "inputs": [
-              {
-                "address": "waf.context.event"
-              },
-              {
-                "address": "server.business_logic.users.login.failure"
-              },
-              {
-                "address": "server.business_logic.users.login.success"
-              }
-            ]
-          }
-        }
-      ],
+      "conditions": [],
       "parameters": {
         "mappings": [
           {
@@ -2063,7 +1995,7 @@
           }
         ]
       },
-      "evaluate": false,
+      "evaluate": true,
       "output": true
     }
   ],
@@ -3090,4 +3022,4 @@
       }
     }
   ]
-}
+}

data/lib/datadog/appsec/component.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'processor'
-require_relative 'processor/rule_merger'
+require_relative 'security_engine/engine'
+require_relative 'security_engine/runner'
 require_relative 'processor/rule_loader'
 require_relative 'actions_handler'
 
@@ -32,7 +32,8 @@ module Datadog
             return
           end
 
-          processor = create_processor(settings, telemetry)
+          require_libddwaf(telemetry: telemetry)
+          Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.appsec.waf_debug
 
           # We want to always instrument user events when AppSec is enabled.
           # There could be cases in which users use the DD_APPSEC_ENABLED Env variable to
@@ -42,67 +43,44 @@ module Datadog
           devise_integration = Datadog::AppSec::Contrib::Devise::Integration.new
           settings.appsec.instrument(:devise) unless devise_integration.patcher.patched?
 
-          new(processor, telemetry)
+          security_engine = SecurityEngine::Engine.new(appsec_settings: settings.appsec, telemetry: telemetry)
+          new(security_engine: security_engine, telemetry: telemetry)
+        rescue
+          Datadog.logger.warn('AppSec is disabled, see logged errors above')
+
+          nil
         end
 
         private
 
-        def create_processor(settings, telemetry)
-          rules = AppSec::Processor::RuleLoader.load_rules(
-            telemetry: telemetry,
-            ruleset: settings.appsec.ruleset
-          )
-          return nil unless rules
-
-          data = AppSec::Processor::RuleLoader.load_data(
-            ip_denylist: settings.appsec.ip_denylist,
-            user_id_denylist: settings.appsec.user_id_denylist,
-          )
-
-          exclusions = AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: settings.appsec.ip_passlist)
-
-          # NOTE: This is a temporary solution before the RuleMerger refactoring
-          #       with new RemoteConfig setup
-          processors = rules['processors']
-          scanners = rules['scanners']
-
-          ruleset = AppSec::Processor::RuleMerger.merge(
-            rules: [rules],
-            data: data,
-            scanners: scanners,
-            processors: processors,
-            exclusions: exclusions,
-            telemetry: telemetry
-          )
-
-          processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
-          return nil unless processor.ready?
-
-          processor
+        def require_libddwaf(telemetry:)
+          require('libddwaf')
+        rescue LoadError => e
+          libddwaf_platform = Gem.loaded_specs['libddwaf']&.platform || 'unknown'
+          ruby_platforms = Gem.platforms.map(&:to_s)
+
+          error_message = "libddwaf failed to load - installed platform: #{libddwaf_platform}, " \
+            "ruby platforms: #{ruby_platforms}"
+
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          telemetry.report(e, description: error_message)
+
+          raise e
         end
       end
 
-      attr_reader :processor, :telemetry
+      attr_reader :security_engine, :telemetry
 
-      def initialize(processor, telemetry)
-        @processor = processor
+      def initialize(security_engine:, telemetry:)
+        @security_engine = security_engine
         @telemetry = telemetry
 
         @mutex = Mutex.new
       end
 
-      def reconfigure(ruleset:, telemetry:)
+      def reconfigure!
         @mutex.synchronize do
-          new_processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
-
-          if new_processor&.ready?
-            old_processor = @processor
-
-            @telemetry = telemetry
-            @processor = new_processor
-
-            old_processor&.finalize
-          end
+          security_engine.reconfigure!
         end
       end
 
@@ -112,10 +90,8 @@ module Datadog
 
       def shutdown!
         @mutex.synchronize do
-          if processor&.ready?
-            processor.finalize
-            @processor = nil
-          end
+          security_engine.finalize!
+          @security_engine = nil
         end
       end
     end

data/lib/datadog/appsec/configuration/settings.rb

@@ -80,16 +80,49 @@ module Datadog
 
               option :ip_passlist do |o|
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The ip_passlist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :ip_denylist do |o|
                 o.type :array
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The ip_denylist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :user_id_denylist do |o|
                 o.type :array
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The user_id_denylist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :waf_timeout do |o|
@@ -311,12 +344,28 @@ module Datadog
               end
 
               settings :api_security do
+                define_method(:enabled?) { get_option(:enabled) }
+
                 option :enabled do |o|
                   o.type :bool
-                  o.env 'DD_EXPERIMENTAL_API_SECURITY_ENABLED'
-                  o.default false
+                  o.env 'DD_API_SECURITY_ENABLED'
+                  o.default true
                 end
 
+                # NOTE: Unfortunately, we have to go with Float due to other libs
+                #       setup, even tho we don't plan to support sub-second delays.
+                #
+                # WARNING: The value will be converted to Integer.
+                option :sample_delay do |o|
+                  o.type :float
+                  o.env 'DD_API_SECURITY_SAMPLE_DELAY'
+                  o.default 30
+                  o.setter do |value|
+                    value.to_i
+                  end
+                end
+
+                # DEV-3.0: Remove `api_security.sample_rate` option
                 option :sample_rate do |o|
                   o.type :float
                   o.env 'DD_API_SECURITY_REQUEST_SAMPLE_RATE'
@@ -325,6 +374,15 @@ module Datadog
                     value = 1 if value > 1
                     SampleRate.new(value)
                   end
+                  o.after_set do |_, _, precedence|
+                    next if precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+
+                    Core.log_deprecation(key: :appsec_api_security_sample_rate) do
+                      'The appsec.api_security.sample_rate setting is deprecated. ' \
+                      'Please remove it from your Datadog.configure block and use ' \
+                      'appsec.api_security.sample_delay instead.'
+                    end
+                  end
                 end
               end
 

data/lib/datadog/appsec/context.rb

@@ -9,6 +9,7 @@ module Datadog
     class Context
       ActiveContextError = Class.new(StandardError)
 
+      # TODO: add delegators for active trace span
       attr_reader :trace, :span, :events
 
       class << self
@@ -20,7 +21,7 @@ module Datadog
         end
 
         def deactivate
-          active&.finalize
+          active&.finalize!
         ensure
           Thread.current[Ext::ACTIVE_CONTEXT_KEY] = nil
         end
@@ -30,12 +31,11 @@ module Datadog
         end
       end
 
-      def initialize(trace, span, security_engine)
+      def initialize(trace, span, waf_runner)
         @trace = trace
         @span = span
         @events = []
-        @security_engine = security_engine
-        @waf_runner = security_engine.new_runner
+        @waf_runner = waf_runner
         @metrics = Metrics::Collector.new
       end
 
@@ -66,8 +66,8 @@ module Datadog
         Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
       end
 
-      def finalize
-        @waf_runner.finalize
+      def finalize!
+        @waf_runner.finalize!
       end
     end
   end

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -92,7 +92,7 @@ module Datadog
             return if value.nil?
             return value.to_s unless anonymize?
 
-            Anonymizer.anonimyze(value.to_s)
+            Anonymizer.anonymize(value.to_s)
           end
 
           def anonymize?

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -7,7 +7,7 @@ require_relative 'gateway/response'
 
 require_relative '../../event'
 require_relative '../../response'
-require_relative '../../processor'
+require_relative '../../api_security'
 require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
@@ -46,7 +46,7 @@ module Datadog
             boot = Datadog::Core::Remote::Tie.boot
             Datadog::Core::Remote::Tie::Tracing.tag(boot, active_span)
 
-            processor = nil
+            security_engine = nil
             ready = false
             ctx = nil
 
@@ -56,11 +56,11 @@ module Datadog
             return @app.call(env) if active_context(env)
 
             Datadog::AppSec.reconfigure_lock do
-              processor = Datadog::AppSec.processor
+              security_engine = Datadog::AppSec.security_engine
 
-              if !processor.nil? && processor.ready?
+              if security_engine
                 ctx = Datadog::AppSec::Context.activate(
-                  Datadog::AppSec::Context.new(active_trace, active_span, processor)
+                  Datadog::AppSec::Context.new(active_trace, active_span, security_engine.new_runner)
                 )
 
                 env[Datadog::AppSec::Ext::CONTEXT_KEY] = ctx
@@ -72,7 +72,7 @@ module Datadog
 
             return @app.call(env) unless ready
 
-            add_appsec_tags(processor, ctx)
+            add_appsec_tags(security_engine, ctx)
             add_request_tags(ctx, env)
 
             http_response = nil
@@ -100,7 +100,23 @@ module Datadog
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
             end
 
-            if AppSec.perform_api_security_check?
+            # NOTE: This is not optimal, but in the current implementation
+            #       `gateway_response` is a container to dispatch response event
+            #       and in case of interruption it suppose to be `nil`.
+            #
+            #       `http_response` is a real response object in both cases, but
+            #       to save us some computations, we will use already pre-computed
+            #       `gateway_response` instead of re-creating it.
+            #
+            # WARNING: This part will be refactored.
+            tmp_response = if interrupt_params
+              Gateway::Response.new(http_response[2], http_response[0], http_response[1], context: ctx)
+            else
+              gateway_response
+            end
+
+            if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
+                AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
               ctx.events.push(
                 AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
               )
@@ -140,7 +156,7 @@ module Datadog
           end
 
           # standard:disable Metrics/MethodLength
-          def add_appsec_tags(processor, context)
+          def add_appsec_tags(security_engine, context)
             span = context.span
             trace = context.trace
 
@@ -150,20 +166,15 @@ module Datadog
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            if processor.diagnostics
-              diagnostics = processor.diagnostics
-
-              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
+            if security_engine.ruleset_version
+              span.set_tag('_dd.appsec.event_rules.version', security_engine.ruleset_version)
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
-                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
+                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(security_engine.waf_addresses))
 
                 # Ensure these tags reach the backend
                 trace.keep!

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -36,22 +36,21 @@ module Datadog
 
             telemetry.report(e, description: 'libddwaf ruleset failed to load')
 
-            nil
+            raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
-            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
+            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
 
             data
           end
 
           def load_exclusions(ip_passlist: [])
-            exclusions = []
-            exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
+            return [] if ip_passlist.empty?
 
-            exclusions
+            passlist_exclusions(ip_passlist)
           end
 
           private

data/lib/datadog/appsec/remote.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_merger'
 require_relative 'processor/rule_loader'
 
 module Datadog
@@ -57,74 +56,35 @@ module Datadog
           remote_features_enabled? ? ASM_PRODUCTS : []
         end
 
-        # rubocop:disable Metrics/MethodLength
-        # rubocop:disable Metrics/CyclomaticComplexity
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
-          # rubocop:disable Metrics/BlockLength
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
-            changes.each do |change|
-              Datadog.logger.debug { "remote config change: '#{change.path}'" }
-            end
-
-            rules = []
-            custom_rules = []
-            data = []
-            overrides = []
-            exclusions = []
-            actions = []
-
-            repository.contents.each do |content|
-              parsed_content = parse_content(content)
-
-              case content.path.product
-              when 'ASM_DD'
-                rules << parsed_content
-              when 'ASM_DATA'
-                data << parsed_content['rules_data'] if parsed_content['rules_data']
-              when 'ASM'
-                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
-                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
-                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-                actions.concat(parsed_content['actions']) if parsed_content['actions']
-              end
-            end
+            next unless AppSec.security_engine
 
-            if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(
-                telemetry: telemetry,
-                ruleset: Datadog.configuration.appsec.ruleset
-              )
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content || change.type == :delete
 
-              raise NoRulesError, 'no default rules available' unless settings_rules
+              case change.type
+              when :insert, :update
+                AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
 
-              rules = [settings_rules]
+                content.applied # steep:ignore
+              when :delete
+                AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
+              end
             end
 
-            ruleset = AppSec::Processor::RuleMerger.merge(
-              rules: rules,
-              data: data,
-              actions: actions,
-              overrides: overrides,
-              exclusions: exclusions,
-              custom_rules: custom_rules,
-              telemetry: telemetry
-            )
-
-            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
-
-            repository.contents.each do |content|
-              content.applied if ASM_PRODUCTS.include?(content.path.product)
-            end
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
           end
-          # rubocop:enable Metrics/BlockLength
 
           [receiver]
         end
-        # rubocop:enable Metrics/MethodLength
-        # rubocop:enable Metrics/CyclomaticComplexity
 
         private