datadog 2.8.0 → 2.39.0

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../instrumentation/gateway'
-require_relative '../../reactive/operation'
-require_relative '../reactive/set_user'
+require_relative "../../event"
+require_relative "../../security_event"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -10,51 +10,77 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
+          ARBITRARY_VALUE = "invalid"
+          EVENT_LOGIN_SUCCESS = "users.login.success"
+          EVENT_LOGIN_FAILURE = "users.login.failure"
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
           class << self
             def watch
               gateway = Instrumentation.gateway
 
               watch_user_id(gateway)
+              watch_user_login(gateway)
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user', :appsec) do |stack, user|
-                block = false
-                event = nil
-                scope = Datadog::AppSec.active_scope
-
-                AppSec::Reactive::Operation.new('identity.set_user') do |op|
-                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: scope.trace,
-                        span: scope.service_entry_span,
-                        user: user,
-                        actions: result.actions
-                      }
-
-                      # We want to keep the trace in case of security event
-                      scope.trace.keep! if scope.trace
-                      Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                      scope.processor_context.events << event
-                    end
-                  end
-
-                  block = Monitor::Reactive::SetUser.publish(op, user)
+              gateway.watch("identity.set_user") do |stack, user|
+                context = AppSec.active_context
+                next stack.call(user) unless context
+
+                context.state[:has_identity_event] = true
+
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
+                  Datadog.logger.debug { "AppSec: skipping WAF check because no user information was provided" }
+                  next stack.call(user)
+                end
+
+                persistent_data = {}
+                persistent_data["usr.id"] = user.id if user.id
+                persistent_data["usr.login"] = user.login if user.login
+                persistent_data["usr.session_id"] = user.session_id if user.session_id
+
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
+                if result.match?
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
                 end
 
-                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
+                stack.call(user)
+              end
+            end
+
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch("appsec.events.user_lifecycle") do |stack, kind|
+                context = AppSec.active_context
+                next stack.call(kind) unless context
 
-                ret, res = stack.call(user)
+                context.state[:has_identity_event] = true
+
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                if result.match?
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
                 end
 
-                [ret, res]
+                stack.call(kind)
               end
             end
           end

data/lib/datadog/appsec/monitor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'monitor/gateway/watcher'
+require_relative "monitor/gateway/watcher"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../assets'
+require_relative "../assets"
 
 module Datadog
   module AppSec
@@ -10,59 +10,56 @@ module Datadog
       module RuleLoader
         class << self
           def load_rules(ruleset:, telemetry:)
-            begin
-              case ruleset
-              when :recommended, :strict
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
-              when :risky
-                Datadog.logger.warn(
-                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                  'The `:recommended` ruleset will be used instead.'\
-                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                )
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-              when String
-                JSON.parse(File.read(File.expand_path(ruleset)))
-              when File, StringIO
-                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
-              when Hash
-                ruleset
-              else
-                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
-              end
-            rescue StandardError => e
-              Datadog.logger.error do
-                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
-              end
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                "The :risky Application Security Management ruleset has been deprecated and no longer available." \
+                "The `:recommended` ruleset will be used instead." \
+                "Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block."
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || "").tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
 
-              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: "libddwaf ruleset failed to load")
 
-              nil
-            end
+            raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
-            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+            data << denylist_data("blocked_ips", ip_denylist) if ip_denylist.any?
+            data << denylist_data("blocked_users", user_id_denylist) if user_id_denylist.any?
 
             data
           end
 
           def load_exclusions(ip_passlist: [])
-            exclusions = []
-            exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
+            return [] if ip_passlist.empty?
 
-            exclusions
+            passlist_exclusions(ip_passlist)
           end
 
           private
 
           def denylist_data(id, denylist)
             {
-              'id' => id,
-              'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+              "id" => id,
+              "type" => "data_with_expiration",
+              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} }
             }
           end
 
@@ -74,46 +71,43 @@ module Datadog
             when Hash
               pass = ip_passlist[:pass]
               monitor = ip_passlist[:monitor]
-            else
-              pass = []
-              monitor = []
             end
 
             exclusions = []
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
+                        "address" => "http.client_ip"
                       }
                     ],
-                    'list' => pass
+                    "list" => pass
                   }
                 }
               ],
-              'id' => SecureRandom.uuid,
+              "id" => SecureRandom.uuid,
             }
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
+                        "address" => "http.client_ip"
                       }
                     ],
-                    'list' => monitor
+                    "list" => monitor
                   }
                 }
               ],
-              'id' => SecureRandom.uuid,
-              'on_match' => 'monitor'
+              "id" => SecureRandom.uuid,
+              "on_match" => "monitor"
             }
 
             exclusions

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../core/rate_limiter'
+require_relative "../core/rate_limiter"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/remote.rb

@@ -1,28 +1,36 @@
 # frozen_string_literal: true
 
-require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_merger'
-require_relative 'processor/rule_loader'
+require_relative "../core/remote/dispatcher"
+require_relative "processor/rule_loader"
 
 module Datadog
   module AppSec
     # Remote
     module Remote
-      class ReadError < StandardError; end
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_PROCESSOR_OVERRIDES = 1 << 16
+        CAP_ASM_CUSTOM_DATA_SCANNERS = 1 << 17
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,13 +43,23 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_PROCESSOR_OVERRIDES,
+          CAP_ASM_CUSTOM_DATA_SCANNERS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
         ].freeze
 
         ASM_PRODUCTS = [
-          'ASM_DD',       # Datadog employee issued configuration
-          'ASM',          # customer issued configuration (rulesets, passlist...)
-          'ASM_FEATURES', # capabilities
-          'ASM_DATA',     # config files (IP addresses or users for blocking)
+          "ASM_DD",       # Datadog employee issued configuration
+          "ASM",          # customer issued configuration (rulesets, passlist...)
+          "ASM_FEATURES", # capabilities
+          "ASM_DATA",     # config files (IP addresses or users for blocking)
         ].freeze
 
         def capabilities
@@ -52,63 +70,42 @@ module Datadog
           remote_features_enabled? ? ASM_PRODUCTS : []
         end
 
-        # rubocop:disable Metrics/MethodLength
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
+            engine = AppSec.security_engine
+            next unless engine
+
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
             changes.each do |change|
-              Datadog.logger.debug { "remote config change: '#{change.path}'" }
-            end
+              next unless change.type == :delete
 
-            rules = []
-            custom_rules = []
-            data = []
-            overrides = []
-            exclusions = []
-
-            repository.contents.each do |content|
-              parsed_content = parse_content(content)
-
-              case content.path.product
-              when 'ASM_DD'
-                rules << parsed_content
-              when 'ASM_DATA'
-                data << parsed_content['rules_data'] if parsed_content['rules_data']
-              when 'ASM'
-                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
-                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
-                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-              end
+              engine.remove_config_at_path(change.path.to_s)
             end
 
-            if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(
-                telemetry: telemetry,
-                ruleset: Datadog.configuration.appsec.ruleset
-              )
-
-              raise NoRulesError, 'no default rules available' unless settings_rules
-
-              rules = [settings_rules]
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content
+
+              case change.type
+              when :insert, :update
+                # @type var content: Core::Remote::Configuration::Content
+                engine.add_or_update_config(parse_content(content), path: change.path.to_s)
+                content.applied
+              end
             end
 
-            ruleset = AppSec::Processor::RuleMerger.merge(
-              rules: rules,
-              data: data,
-              overrides: overrides,
-              exclusions: exclusions,
-              custom_rules: custom_rules,
-              telemetry: telemetry
-            )
-
-            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
           end
 
           [receiver]
         end
-        # rubocop:enable Metrics/MethodLength
 
         private
 
@@ -117,13 +114,7 @@ module Datadog
         end
 
         def parse_content(content)
-          data = content.data.read
-
-          content.data.rewind
-
-          raise ReadError, 'EOF reached' if data.nil?
-
-          JSON.parse(data)
+          JSON.parse(content.data)
         end
       end
     end