datadog 2.7.1 → 2.17.0

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
-require_relative '../../reactive/operation'
-require_relative '../reactive/set_user'
 
 module Datadog
   module AppSec
@@ -10,51 +10,71 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
+          ARBITRARY_VALUE = 'invalid'
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
           class << self
             def watch
               gateway = Instrumentation.gateway
 
               watch_user_id(gateway)
+              watch_user_login(gateway)
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                block = false
-                event = nil
-                scope = Datadog::AppSec.active_scope
-
-                AppSec::Reactive::Operation.new('identity.set_user') do |op|
-                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: scope.trace,
-                        span: scope.service_entry_span,
-                        user: user,
-                        actions: result.actions
-                      }
-
-                      # We want to keep the trace in case of security event
-                      scope.trace.keep! if scope.trace
-                      Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                      scope.processor_context.events << event
-                    end
-                  end
-
-                  block = Monitor::Reactive::SetUser.publish(op, user)
+                context = AppSec.active_context
+
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
+
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
+                persistent_data['usr.session_id'] = user.session_id if user.session_id
+
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
+                if result.match?
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
                 end
 
-                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
+                stack.call(user)
+              end
+            end
+
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+                context = AppSec.active_context
 
-                ret, res = stack.call(user)
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                if result.match?
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
                 end
 
-                [ret, res]
+                stack.call(kind)
               end
             end
           end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -10,35 +10,33 @@ module Datadog
       module RuleLoader
         class << self
           def load_rules(ruleset:, telemetry:)
-            begin
-              case ruleset
-              when :recommended, :strict
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
-              when :risky
-                Datadog.logger.warn(
-                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                  'The `:recommended` ruleset will be used instead.'\
-                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                )
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-              when String
-                JSON.parse(File.read(File.expand_path(ruleset)))
-              when File, StringIO
-                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
-              when Hash
-                ruleset
-              else
-                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
-              end
-            rescue StandardError => e
-              Datadog.logger.error do
-                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
-              end
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
+                'The `:recommended` ruleset will be used instead.' \
+                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
 
-              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: 'libddwaf ruleset failed to load')
 
-              nil
-            end
+            nil
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
@@ -62,7 +60,7 @@ module Datadog
             {
               'id' => id,
               'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
             }
           end
 
@@ -74,9 +72,6 @@ module Datadog
             when Hash
               pass = ip_passlist[:pass]
               monitor = ip_passlist[:monitor]
-            else
-              pass = []
-              monitor = []
             end
 
             exclusions = []

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -11,8 +11,8 @@ module Datadog
         # RuleVersionMismatchError
         class RuleVersionMismatchError < StandardError
           def initialize(version1, version2)
-            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
-              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+            msg = 'Merging rule files with different version could lead to unkown behaviour. ' \
+              "We have receieve two rule files with versions: #{version1}, #{version2}. " \
               'Please validate the configuration is correct and try again.'
             super(msg)
           end
@@ -22,12 +22,12 @@ module Datadog
           # TODO: `processors` and `scanners` are not provided by the caller, consider removing them
           def merge(
             telemetry:,
-            rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
+            rules:, actions: [], data: [], overrides: [], exclusions: [], custom_rules: [],
             processors: nil, scanners: nil
           )
             processors ||= begin
               default_waf_processors
-            rescue StandardError => e
+            rescue => e
               Datadog.logger.error("libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}")
               telemetry.report(
                 e,
@@ -38,7 +38,7 @@ module Datadog
 
             scanners ||= begin
               default_waf_scanners
-            rescue StandardError => e
+            rescue => e
               Datadog.logger.error("libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}")
               telemetry.report(
                 e,
@@ -54,6 +54,7 @@ module Datadog
             combined_exclusions = combine_exclusions(exclusions) if exclusions.any?
             combined_custom_rules = combine_custom_rules(custom_rules) if custom_rules.any?
 
+            combined_rules['actions'] = actions if actions.any?
             combined_rules['rules_data'] = combined_data if combined_data
             combined_rules['rules_override'] = combined_overrides if combined_overrides
             combined_rules['exclusions'] = combined_exclusions if combined_exclusions
@@ -145,7 +146,7 @@ module Datadog
             end
 
             result.each_with_object([]) do |entry, acc|
-              value = { 'value' => entry[0] }
+              value = {'value' => entry[0]}
               value['expiration'] = entry[1] if entry[1]
 
               acc << value

data/lib/datadog/appsec/processor.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'processor/context'
+require_relative 'security_engine/runner'
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
+    # NOTE: This class will be moved under AppSec::SecurityEngine namespace
     class Processor
       attr_reader :diagnostics, :addresses
 
@@ -29,8 +30,8 @@ module Datadog
         @handle.finalize
       end
 
-      def new_context
-        Context.new(@handle, telemetry: @telemetry)
+      def new_runner
+        SecurityEngine::Runner.new(@handle, telemetry: @telemetry)
       end
 
       private
@@ -81,7 +82,7 @@ module Datadog
         @diagnostics = e.diagnostics if e.diagnostics
 
         false
-      rescue StandardError => e
+      rescue => e
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end

data/lib/datadog/appsec/remote.rb

@@ -9,20 +9,23 @@ module Datadog
     # Remote
     module Remote
       class ReadError < StandardError; end
+
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
+        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
+        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
+        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
+        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
+        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
+        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
+        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
+        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
+        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,6 +38,8 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -53,10 +58,12 @@ module Datadog
         end
 
         # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/CyclomaticComplexity
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
+          # rubocop:disable Metrics/BlockLength
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
             changes.each do |change|
               Datadog.logger.debug { "remote config change: '#{change.path}'" }
@@ -99,18 +106,25 @@ module Datadog
             ruleset = AppSec::Processor::RuleMerger.merge(
               rules: rules,
               data: data,
+              actions: actions,
               overrides: overrides,
               exclusions: exclusions,
               custom_rules: custom_rules,
               telemetry: telemetry
             )
 
-            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
+            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
+
+            repository.contents.each do |content|
+              content.applied if ASM_PRODUCTS.include?(content.path.product)
+            end
           end
+          # rubocop:enable Metrics/BlockLength
 
           [receiver]
         end
         # rubocop:enable Metrics/MethodLength
+        # rubocop:enable Metrics/CyclomaticComplexity
 
         private
 

data/lib/datadog/appsec/response.rb

@@ -19,104 +19,38 @@ module Datadog
         [status, headers, body]
       end
 
-      def to_sinatra_response
-        ::Sinatra::Response.new(body, status, headers)
-      end
-
-      def to_action_dispatch_response
-        ::ActionDispatch::Response.new(status, headers, body)
-      end
-
       class << self
-        def negotiate(env, actions)
-          # @type var configured_response: Response?
-          configured_response = nil
-          actions.each do |action|
-            # Need to use next to make steep happy :(
-            # I rather use break to stop the execution
-            next if configured_response
-
-            action_configuration = AppSec::Processor::Actions.fetch_configuration(action)
-            next unless action_configuration
-
-            configured_response = case action_configuration['type']
-                                  when 'block_request'
-                                    block_response(env, action_configuration['parameters'])
-                                  when 'redirect_request'
-                                    redirect_response(env, action_configuration['parameters'])
-                                  end
-          end
+        def from_interrupt_params(interrupt_params, http_accept_header)
+          return redirect_response(interrupt_params) if interrupt_params['location']
 
-          configured_response || default_response(env)
-        end
-
-        def graphql_response(gateway_multiplex)
-          multiplex_return = []
-          gateway_multiplex.queries.each do |query|
-            # This method is only called in places where GraphQL-Ruby is already required
-            query_result = ::GraphQL::Query::Result.new(
-              query: query,
-              values: JSON.parse(content('application/json'))
-            )
-            multiplex_return << query_result
-          end
-
-          multiplex_return
+          block_response(interrupt_params, http_accept_header)
         end
 
         private
 
-        def default_response(env)
-          content_type = content_type(env)
-
-          body = []
-          body << content(content_type)
+        def block_response(interrupt_params, http_accept_header)
+          content_type = case interrupt_params['type']
+          when nil, 'auto' then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          end
 
           Response.new(
-            status: 403,
-            headers: { 'Content-Type' => content_type },
-            body: body,
+            status: interrupt_params['status_code']&.to_i || 403,
+            headers: {'Content-Type' => content_type},
+            body: [content(content_type)],
           )
         end
 
-        def block_response(env, options)
-          content_type = if options['type'] == 'auto'
-                           content_type(env)
-                         else
-                           FORMAT_TO_CONTENT_TYPE[options['type']]
-                         end
-
-          body = []
-          body << content(content_type)
+        def redirect_response(interrupt_params)
+          status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: options['status_code'] || 403,
-            headers: { 'Content-Type' => content_type },
-            body: body,
+            status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
+            headers: {'Location' => interrupt_params.fetch('location')},
+            body: [],
           )
         end
 
-        def redirect_response(env, options)
-          if options['location'] && !options['location'].empty?
-            content_type = content_type(env)
-
-            status = options['status_code'] >= 300 && options['status_code'] < 400 ? options['status_code'] : 303
-
-            headers = {
-              'Content-Type' => content_type,
-              'Location' => options['location']
-            }
-
-            Response.new(
-              status: status,
-              headers: headers,
-              body: [],
-            )
-          else
-            default_response(env)
-          end
-        end
-
         CONTENT_TYPE_TO_FORMAT = {
           'application/json' => :json,
           'text/html' => :html,
@@ -130,10 +64,10 @@ module Datadog
 
         DEFAULT_CONTENT_TYPE = 'application/json'
 
-        def content_type(env)
-          return DEFAULT_CONTENT_TYPE unless env.key?('HTTP_ACCEPT')
+        def content_type(http_accept_header)
+          return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = env['HTTP_ACCEPT'].split(',').map(&:strip)
+          accept_types = http_accept_header.split(',').map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+            @events = events
+            @actions = actions
+            @derivatives = derivatives
+
+            @timeout = timeout
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            !!@timeout
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:)
+            @events = []
+            @actions = @derivatives = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            false
+          end
+
+          def match?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A class that check input via security engine (WAF) and respond with result.
+      class Runner
+        SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        def initialize(handle, telemetry:)
+          @mutex = Mutex.new
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? || v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? || v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          report_execution(result)
+
+          unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+          end
+
+          klass = (result.status == :match) ? Result::Match : Result::Ok
+          klass.new(
+            events: result.events,
+            actions: result.actions,
+            derivatives: result.derivatives,
+            timeout: result.timeout,
+            duration_ns: result.total_runtime,
+            duration_ext_ns: (stop_ns - start_ns)
+          )
+        ensure
+          @mutex.unlock
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A namespace for secutiry library we use to detect and prevent threats.
+    module SecurityEngine
+    end
+  end
+end