datadog 2.12.0 → 2.22.0

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,35 +1,29 @@
 # frozen_string_literal: true
 
+require_relative 'gateway/middleware'
+
 module Datadog
   module AppSec
     # Instrumentation for AppSec
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
-        # Instrumentation gateway middleware
-        class Middleware
-          attr_reader :key, :block
-
-          def initialize(key, &block)
-            @key = key
-            @block = block
-          end
-
-          def call(stack, env)
-            @block.call(stack, env)
-          end
-        end
-
-        private_constant :Middleware
-
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
+          @pushed_events = {}
         end
 
+        # NOTE: Be careful with pushed names because every pushed event name
+        #       is recorded in order to provide an ability to any subscriber
+        #       to check wether an arbitrary event had happened.
+        #
+        # WARNING: If we start pushing generated names we should consider
+        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          block ||= -> {}
+          @pushed_events[name] = true
 
-          middlewares_for_name = middlewares[name]
+          block ||= -> {}
+          middlewares_for_name = @middlewares[name]
 
           return [block.call, nil] if middlewares_for_name.empty?
 
@@ -48,14 +42,15 @@ module Datadog
         end
 
         def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
         end
 
-        private
-
-        attr_reader :middlewares
+        def pushed?(name)
+          @pushed_events.key?(name)
+        end
       end
 
+      # NOTE: This left as-is and will be depricated soon.
       def self.gateway
         @gateway ||= Gateway.new # TODO: not thread safe
       end

data/lib/datadog/appsec/metrics/collector.rb

@@ -5,31 +5,51 @@ module Datadog
     module Metrics
       # A class responsible for collecting WAF and RASP call metrics.
       class Collector
-        Store = Struct.new(:evals, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
+        Store = Struct.new(
+          :evals,
+          :matches,
+          :errors,
+          :timeouts,
+          :duration_ns,
+          :duration_ext_ns,
+          :inputs_truncated,
+          keyword_init: true
+        )
 
         attr_reader :waf, :rasp
 
         def initialize
           @mutex = Mutex.new
-          @waf = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
-          @rasp = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+
+          @waf = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
+          @rasp = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
         end
 
         def record_waf(result)
           @mutex.synchronize do
             @waf.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @waf.timeouts += 1 if result.timeout?
             @waf.duration_ns += result.duration_ns
             @waf.duration_ext_ns += result.duration_ext_ns
+            @waf.inputs_truncated += 1 if result.input_truncated?
           end
         end
 
         def record_rasp(result)
           @mutex.synchronize do
             @rasp.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @rasp.timeouts += 1 if result.timeout?
             @rasp.duration_ns += result.duration_ns
             @rasp.duration_ext_ns += result.duration_ext_ns
+            @rasp.inputs_truncated += 1 if result.input_truncated?
           end
         end
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -8,9 +8,9 @@ module Datadog
         module_function
 
         def report_rasp(type, result)
-          return if result.is_a?(SecurityEngine::Result::Error)
+          return if result.error?
 
-          tags = { rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING }
+          tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
           AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF request metrics via Telemetry.
+      module TelemetryExporter
+        module_function
+
+        def export_waf_request_metrics(metrics, context)
+          AppSec.telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            tags: {
+              waf_version: WAF::VERSION::BASE_STRING,
+              event_rules_version: context.waf_runner_ruleset_version,
+              rule_triggered: metrics.matches.positive?.to_s,
+              waf_error: metrics.errors.positive?.to_s,
+              waf_timeout: metrics.timeouts.positive?.to_s,
+              request_blocked: context.interrupted?.to_s,
+              block_failure: 'false',
+              rate_limited: (!context.trace.sampled?).to_s,
+              input_truncated: metrics.inputs_truncated.positive?.to_s,
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics.rb

@@ -11,3 +11,4 @@ end
 require_relative 'metrics/collector'
 require_relative 'metrics/exporter'
 require_relative 'metrics/telemetry'
+require_relative 'metrics/telemetry_exporter'

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
 module Datadog
@@ -8,38 +10,71 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
+          ARBITRARY_VALUE = 'invalid'
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
           class << self
             def watch
               gateway = Instrumentation.gateway
 
               watch_user_id(gateway)
+              watch_user_login(gateway)
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                context = Datadog::AppSec.active_context
+                context = AppSec.active_context
+
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
 
-                persistent_data = {
-                  'usr.id' => user.id
-                }
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
+                persistent_data['usr.session_id'] = user.session_id if user.session_id
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
                 if result.match?
-                  Datadog::AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
 
-                  context.events << {
-                    waf_result: result,
-                    trace: context.trace,
-                    span: context.span,
-                    user: user,
-                    actions: result.actions
-                  }
+                stack.call(user)
+              end
+            end
 
-                  Datadog::AppSec::ActionsHandler.handle(result.actions)
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+                context = AppSec.active_context
+
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
                 end
 
-                stack.call(user)
+                if result.match?
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(kind)
               end
             end
           end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -10,50 +10,47 @@ module Datadog
       module RuleLoader
         class << self
           def load_rules(ruleset:, telemetry:)
-            begin
-              case ruleset
-              when :recommended, :strict
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
-              when :risky
-                Datadog.logger.warn(
-                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                  'The `:recommended` ruleset will be used instead.'\
-                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                )
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-              when String
-                JSON.parse(File.read(File.expand_path(ruleset)))
-              when File, StringIO
-                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
-              when Hash
-                ruleset
-              else
-                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
-              end
-            rescue StandardError => e
-              Datadog.logger.error do
-                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
-              end
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
+                'The `:recommended` ruleset will be used instead.' \
+                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
 
-              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: 'libddwaf ruleset failed to load')
 
-              nil
-            end
+            raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
-            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
+            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
 
             data
           end
 
           def load_exclusions(ip_passlist: [])
-            exclusions = []
-            exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
+            return [] if ip_passlist.empty?
 
-            exclusions
+            passlist_exclusions(ip_passlist)
           end
 
           private
@@ -62,7 +59,7 @@ module Datadog
             {
               'id' => id,
               'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
             }
           end
 

data/lib/datadog/appsec/remote.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_merger'
 require_relative 'processor/rule_loader'
 
 module Datadog
@@ -9,20 +8,29 @@ module Datadog
     # Remote
     module Remote
       class ReadError < StandardError; end
+
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,6 +43,14 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -52,67 +68,35 @@ module Datadog
           remote_features_enabled? ? ASM_PRODUCTS : []
         end
 
-        # rubocop:disable Metrics/MethodLength
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
-            changes.each do |change|
-              Datadog.logger.debug { "remote config change: '#{change.path}'" }
-            end
-
-            rules = []
-            custom_rules = []
-            data = []
-            overrides = []
-            exclusions = []
-
-            repository.contents.each do |content|
-              parsed_content = parse_content(content)
-
-              case content.path.product
-              when 'ASM_DD'
-                rules << parsed_content
-              when 'ASM_DATA'
-                data << parsed_content['rules_data'] if parsed_content['rules_data']
-              when 'ASM'
-                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
-                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
-                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-              end
-            end
+            next unless AppSec.security_engine
 
-            if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(
-                telemetry: telemetry,
-                ruleset: Datadog.configuration.appsec.ruleset
-              )
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content || change.type == :delete
 
-              raise NoRulesError, 'no default rules available' unless settings_rules
+              case change.type
+              when :insert, :update
+                AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
 
-              rules = [settings_rules]
+                content.applied # steep:ignore
+              when :delete
+                AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
+              end
             end
 
-            ruleset = AppSec::Processor::RuleMerger.merge(
-              rules: rules,
-              data: data,
-              overrides: overrides,
-              exclusions: exclusions,
-              custom_rules: custom_rules,
-              telemetry: telemetry
-            )
-
-            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
-
-            repository.contents.each do |content|
-              content.applied if ASM_PRODUCTS.include?(content.path.product)
-            end
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
           end
 
           [receiver]
         end
-        # rubocop:enable Metrics/MethodLength
 
         private
 

data/lib/datadog/appsec/response.rb

@@ -30,13 +30,13 @@ module Datadog
 
         def block_response(interrupt_params, http_accept_header)
           content_type = case interrupt_params['type']
-                         when nil, 'auto' then content_type(http_accept_header)
-                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
-                         end
+          when nil, 'auto' then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          end
 
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
+            headers: {'Content-Type' => content_type},
             body: [content(content_type)],
           )
         end
@@ -45,8 +45,8 @@ module Datadog
           status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-            headers: { 'Location' => interrupt_params.fetch('location') },
+            status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
+            headers: {'Location' => interrupt_params.fetch('location')},
             body: [],
           )
         end