datadog 2.21.0 → 2.38.0

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -10,18 +10,9 @@ module Datadog
       class Gateway
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
-          @pushed_events = {}
         end
 
-        # NOTE: Be careful with pushed names because every pushed event name
-        #       is recorded in order to provide an ability to any subscriber
-        #       to check wether an arbitrary event had happened.
-        #
-        # WARNING: If we start pushing generated names we should consider
-        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          @pushed_events[name] = true
-
           block ||= -> {}
           middlewares_for_name = @middlewares[name]
 
@@ -41,12 +32,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
-        end
-
-        def pushed?(name)
-          @pushed_events.key?(name)
+        def watch(name, &block)
+          @middlewares[name] << Middleware.new(&block)
         end
       end
 

data/lib/datadog/appsec/metrics/collector.rb

@@ -5,14 +5,45 @@ module Datadog
     module Metrics
       # A class responsible for collecting WAF and RASP call metrics.
       class Collector
-        Store = Struct.new(:evals, :matches, :errors, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
+        Store = Struct.new(
+          :evals,
+          :matches,
+          :errors,
+          :timeouts,
+          :duration_ns,
+          :duration_ext_ns,
+          :inputs_truncated,
+          :downstream_requests,
+          keyword_init: true
+        )
 
-        attr_reader :waf, :rasp
+        DownstreamResponseStore = Struct.new(
+          :content_type_invalid,
+          :content_length_missing,
+          :content_length_too_big,
+          :content_exceed_content_length,
+          keyword_init: true
+        )
+
+        attr_reader :waf, :rasp, :downstream_responses
 
         def initialize
           @mutex = Mutex.new
-          @waf = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
-          @rasp = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+
+          @waf = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
+            duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
+          )
+
+          @rasp = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
+            duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
+          )
+
+          @downstream_responses = DownstreamResponseStore.new(
+            content_type_invalid: 0, content_length_missing: 0,
+            content_length_too_big: 0, content_exceed_content_length: 0
+          )
         end
 
         def record_waf(result)
@@ -23,10 +54,11 @@ module Datadog
             @waf.timeouts += 1 if result.timeout?
             @waf.duration_ns += result.duration_ns
             @waf.duration_ext_ns += result.duration_ext_ns
+            @waf.inputs_truncated += 1 if result.input_truncated?
           end
         end
 
-        def record_rasp(result)
+        def record_rasp(result, type:, phase: nil)
           @mutex.synchronize do
             @rasp.evals += 1
             @waf.matches += 1 if result.match?
@@ -34,8 +66,14 @@ module Datadog
             @rasp.timeouts += 1 if result.timeout?
             @rasp.duration_ns += result.duration_ns
             @rasp.duration_ext_ns += result.duration_ext_ns
+            @rasp.inputs_truncated += 1 if result.input_truncated?
+            @rasp.downstream_requests += 1 if type == Ext::RASP_SSRF && phase == Ext::RASP_REQUEST_PHASE
           end
         end
+
+        def record_ignored_downstream_response_body(reason)
+          @mutex.synchronize { @downstream_responses[reason] += 1 }
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics/exporter.rb

@@ -3,7 +3,7 @@
 module Datadog
   module AppSec
     module Metrics
-      # A class responsible for exporting WAF and RASP call metrics.
+      # A class responsible for exporting WAF and RASP call metrics
       module Exporter
         module_function
 
@@ -22,11 +22,24 @@ module Datadog
           span.set_tag('_dd.appsec.rasp.timeout', 1) unless metrics.timeouts.zero?
           span.set_tag('_dd.appsec.rasp.duration', convert_ns_to_us(metrics.duration_ns))
           span.set_tag('_dd.appsec.rasp.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+
+          # NOTE: In case of downstream requests being analyzed additionally
+          #       with `Context.run_waf` method, we would need to share it
+          #       between two exporting methods
+          unless metrics.downstream_requests.zero?
+            span.set_tag('_dd.appsec.downstream_request', metrics.downstream_requests)
+          end
         end
 
-        # private
+        def export_downstream_response_metrics(metrics, span)
+          metrics.each_pair do |reason, count|
+            next if count.zero?
+
+            span.set_tag("_dd.appsec.downstream_request.response_body_ignored.#{reason}", count)
+          end
+        end
 
-        def convert_ns_to_us(value)
+        private_class_method def convert_ns_to_us(value)
           value / 1000.0
         end
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -5,17 +5,34 @@ module Datadog
     module Metrics
       # A class responsible for reporting WAF and RASP telemetry metrics.
       module Telemetry
+        ACTION_BLOCK = 'block_request'
+        ACTION_REDIRECT = 'redirect_request'
+        BLOCK_SUCCESS = 'success'
+        BLOCK_IRRELEVANT = 'irrelevant'
+
         module_function
 
-        def report_rasp(type, result)
+        def report_rasp(type, result, phase: nil)
           return if result.error?
 
-          tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
+          tags = {rule_type: type, waf_version: WAF::VERSION::BASE_STRING}
+          tags[:rule_variant] = phase if phase
+
+          context = AppSec.active_context
+          tags[:event_rules_version] = context.waf_runner_ruleset_version if context
+
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
           AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)
-          AppSec.telemetry.inc(namespace, 'rasp.rule.match', 1, tags: tags) if result.match?
           AppSec.telemetry.inc(namespace, 'rasp.timeout', 1, tags: tags) if result.timeout?
+
+          if result.match?
+            blocked = result.actions.key?(ACTION_BLOCK) || result.actions.key?(ACTION_REDIRECT)
+            # NOTE: Mutates tags to avoid an extra hash allocation. Keep this the last .inc call.
+            tags[:block] = blocked ? BLOCK_SUCCESS : BLOCK_IRRELEVANT
+
+            AppSec.telemetry.inc(namespace, 'rasp.rule.match', 1, tags: tags)
+          end
         end
       end
     end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -18,10 +18,29 @@ module Datadog
               waf_timeout: metrics.timeouts.positive?.to_s,
               request_blocked: context.interrupted?.to_s,
               block_failure: 'false',
-              rate_limited: (!context.trace.sampled?).to_s
+              rate_limited: (!context.trace.sampled?).to_s,
+              input_truncated: metrics.inputs_truncated.positive?.to_s,
             }
           )
         end
+
+        def export_api_security_metrics(context)
+          web_framework = context.state[:web_framework]
+          return unless web_framework
+
+          if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
+            AppSec.telemetry.inc(
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, 'api_security.missing_route', 1,
+              tags: {framework: web_framework}
+            )
+          end
+
+          metric_name = context.state[:schema_extracted] ? 'schema' : 'no_schema'
+          AppSec.telemetry.inc(
+            AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
+            tags: {framework: web_framework}
+          )
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+require_relative 'metrics/collector'
+require_relative 'metrics/exporter'
+require_relative 'metrics/telemetry'
+require_relative 'metrics/telemetry_exporter'
+
 module Datadog
   module AppSec
     # This namespace contains classes related to metrics collection and exportation.
@@ -7,8 +12,3 @@ module Datadog
     end
   end
 end
-
-require_relative 'metrics/collector'
-require_relative 'metrics/exporter'
-require_relative 'metrics/telemetry'
-require_relative 'metrics/telemetry_exporter'

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -24,8 +24,9 @@ module Datadog
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user', :appsec) do |stack, user|
+              gateway.watch('identity.set_user') do |stack, user|
                 context = AppSec.active_context
+                context.state[:has_identity_event] = true
 
                 if user.id.nil? && user.login.nil? && user.session_id.nil?
                   Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
@@ -39,14 +40,14 @@ module Datadog
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                if result.match? || result.derivatives.any?
+                if result.match? || result.attributes.any?
                   context.events.push(
                     AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                   )
                 end
 
                 if result.match?
-                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag(context, result)
                   AppSec::ActionsHandler.handle(result.actions)
                 end
 
@@ -55,22 +56,23 @@ module Datadog
             end
 
             def watch_user_login(gateway = Instrumentation.gateway)
-              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+              gateway.watch('appsec.events.user_lifecycle') do |stack, kind|
                 context = AppSec.active_context
+                context.state[:has_identity_event] = true
 
                 next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
 
                 persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                if result.match? || result.derivatives.any?
+                if result.match? || result.attributes.any?
                   context.events.push(
                     AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                   )
                 end
 
                 if result.match?
-                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag(context, result)
                   AppSec::ActionsHandler.handle(result.actions)
                 end
 

data/lib/datadog/appsec/remote.rb

@@ -7,24 +7,30 @@ module Datadog
   module AppSec
     # Remote
     module Remote
-      class ReadError < StandardError; end
-
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
-        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
-        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_PROCESSOR_OVERRIDES = 1 << 16
+        CAP_ASM_CUSTOM_DATA_SCANNERS = 1 << 17
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -37,8 +43,16 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_PROCESSOR_OVERRIDES,
+          CAP_ASM_CUSTOM_DATA_SCANNERS,
           CAP_ASM_RASP_SSRF,
           CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -61,19 +75,26 @@ module Datadog
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
-            next unless AppSec.security_engine
+            engine = AppSec.security_engine
+            next unless engine
+
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
+            changes.each do |change|
+              next unless change.type == :delete
+
+              engine.remove_config_at_path(change.path.to_s)
+            end
 
             changes.each do |change|
               content = repository[change.path]
-              next unless content || change.type == :delete
+              next unless content
 
               case change.type
               when :insert, :update
-                AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
-
-                content.applied # steep:ignore
-              when :delete
-                AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
+                # @type var content: Core::Remote::Configuration::Content
+                engine.add_or_update_config(parse_content(content), path: change.path.to_s)
+                content.applied
               end
             end
 
@@ -93,13 +114,7 @@ module Datadog
         end
 
         def parse_content(content)
-          data = content.data.read
-
-          content.data.rewind
-
-          raise ReadError, 'EOF reached' if data.nil?
-
-          JSON.parse(data)
+          JSON.parse(content.data)
         end
       end
     end

data/lib/datadog/appsec/response.rb

@@ -7,6 +7,8 @@ module Datadog
   module AppSec
     # AppSec response
     class Response
+      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
+
       attr_reader :status, :headers, :body
 
       def initialize(status:, headers: {}, body: [])
@@ -37,16 +39,26 @@ module Datadog
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
             headers: {'Content-Type' => content_type},
-            body: [content(content_type)],
+            body: [
+              content(
+                security_response_id: interrupt_params['security_response_id'],
+                content_type: content_type
+              )
+            ],
           )
         end
 
         def redirect_response(interrupt_params)
           status_code = interrupt_params['status_code'].to_i
+          location = interrupt_params.fetch('location')
+
+          if (security_response_id = interrupt_params.fetch('security_response_id'))
+            location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
+          end
 
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => interrupt_params.fetch('location')},
+            headers: {'Location' => location},
             body: [],
           )
         end
@@ -82,16 +94,18 @@ module Datadog
           DEFAULT_CONTENT_TYPE
         end
 
-        def content(content_type)
+        def content(security_response_id:, content_type:)
           content_format = CONTENT_TYPE_TO_FORMAT[content_type]
 
           using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
 
-          if using_default
+          template = if using_default
             Datadog::AppSec::Assets.blocked(format: content_format)
           else
             Datadog.configuration.appsec.block.templates.send(content_format)
           end
+
+          template.gsub(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id.to_s)
         end
       end
     end

data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative 'route_text'
+require_relative 'route_pattern'
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes Rails route patterns into route format, inspired by
+      # OpenAPI v3 path templating
+      #
+      # NOTE: Uses the parsed Journey AST when available
+      #
+      # @api private
+      class RailsRoutePattern
+        DOT_CHAR = '.'
+        GROUP_OPEN_CHAR = '('
+        OPTIONAL_GROUP_PATTERN = /\(([^()]*)\)/
+        NAMED_PARAM_PREFIX_CHAR = ':'
+        GLOB_PARAM_PREFIX_CHAR = '*'
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path_params:, path:)
+          @path_params = path_params
+          @path = path
+
+          if @pattern.is_a?(String)
+            # NOTE: Journey groups without route params are never kept
+            #       Example: `/foo(/bar)` with request `/foo/bar` normalizes to `/foo`
+            pattern = remove_paramless_optional_groups(@pattern)
+            return RoutePattern.new(pattern).normalize(path: path)
+          end
+
+          route_path = @pattern.path
+          route_spec = route_path.spec
+
+          unless route_spec.respond_to?(:type)
+            return RoutePattern.new(route_spec.to_s).normalize(path: path)
+          end
+
+          if route_path.names.empty?
+            route_string = route_spec.to_s
+            return RouteText.escape(route_string) unless route_string.include?(GROUP_OPEN_CHAR)
+          end
+
+          buffer = Buffer.new
+          visit_route_node(route_spec, buffer)
+          buffer.to_path
+        end
+
+        private
+
+        def remove_paramless_optional_groups(pattern)
+          result = pattern
+
+          loop do
+            substituted = result.gsub(OPTIONAL_GROUP_PATTERN) do
+              # NOTE: OPTIONAL_GROUP_PATTERN always captures a string for each gsub match
+              group = ::Regexp.last_match(1) # : String
+              optional_group_has_route_params?(group) ? "(#{group})" : ''
+            end
+
+            return result if substituted == result
+
+            result = substituted
+          end
+        end
+
+        def visit_route_node(node, buffer)
+          case node.type
+          when :CAT
+            visit_route_node(node.left, buffer)
+            visit_route_node(node.right, buffer)
+          when :SLASH
+            buffer.flush
+          when :LITERAL
+            buffer.add_literal(node.left)
+          when :DOT
+            buffer.add_literal(DOT_CHAR)
+          when :SYMBOL, :STAR
+            buffer.add_param(node.name)
+          when :GROUP
+            visit_route_node(node.left, buffer) if group_present?(node.left)
+          end
+        end
+
+        def group_present?(node)
+          param_names = collect_group_param_names(node, [])
+          return false if param_names.empty?
+
+          param_names.all? { |name| param_matched_path?(name.to_sym) }
+        end
+
+        def param_matched_path?(name)
+          return @path_params[name].is_a?(String) unless name == :format
+
+          format = @path_params[:format]
+          return false if !format.is_a?(String) || format.empty?
+
+          dot_index = @path.length - format.length - 1
+          return false if dot_index < 0 || @path[dot_index] != DOT_CHAR
+
+          @path.end_with?(format)
+        end
+
+        def collect_group_param_names(node, memo)
+          case node.type
+          when :SYMBOL, :STAR
+            memo << node.name
+          when :CAT
+            collect_group_param_names(node.left, memo)
+            collect_group_param_names(node.right, memo)
+          end
+
+          memo
+        end
+
+        def optional_group_has_route_params?(group)
+          group.include?(NAMED_PARAM_PREFIX_CHAR) || group.include?(GLOB_PARAM_PREFIX_CHAR)
+        end
+
+        class Buffer
+          def initialize
+            @segments = []
+            @text = +''
+            @params = []
+            @nameless_param_count = 0
+          end
+
+          def add_literal(text)
+            @text << text
+          end
+
+          def add_param(name)
+            @params << name
+          end
+
+          def flush
+            return if @text.empty? && @params.empty?
+
+            @segments << if @params.empty?
+              RouteText.escape(@text)
+            else
+              render_params(@params)
+            end
+
+            @text = +''
+            @params.clear
+          end
+
+          def to_path
+            flush
+
+            "/#{@segments.join('/')}"
+          end
+
+          private
+
+          def render_params(params)
+            names = params.map do |name| # $ String
+              next name unless name.empty?
+
+              @nameless_param_count += 1
+              "param#{@nameless_param_count}"
+            end
+
+            "{#{names.join('+')}}"
+          end
+        end
+      end
+    end
+  end
+end