datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/remote.rb

@@ -1,31 +1,36 @@
 # frozen_string_literal: true
 
 require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_merger'
 require_relative 'processor/rule_loader'
 
 module Datadog
   module AppSec
     # Remote
     module Remote
-      class ReadError < StandardError; end
-
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
-        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
-        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_PROCESSOR_OVERRIDES = 1 << 16
+        CAP_ASM_CUSTOM_DATA_SCANNERS = 1 << 17
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -38,8 +43,16 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_PROCESSOR_OVERRIDES,
+          CAP_ASM_CUSTOM_DATA_SCANNERS,
           CAP_ASM_RASP_SSRF,
           CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -57,74 +70,42 @@ module Datadog
           remote_features_enabled? ? ASM_PRODUCTS : []
         end
 
-        # rubocop:disable Metrics/MethodLength
-        # rubocop:disable Metrics/CyclomaticComplexity
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
-          # rubocop:disable Metrics/BlockLength
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
+            engine = AppSec.security_engine
+            next unless engine
+
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
             changes.each do |change|
-              Datadog.logger.debug { "remote config change: '#{change.path}'" }
-            end
+              next unless change.type == :delete
 
-            rules = []
-            custom_rules = []
-            data = []
-            overrides = []
-            exclusions = []
-            actions = []
-
-            repository.contents.each do |content|
-              parsed_content = parse_content(content)
-
-              case content.path.product
-              when 'ASM_DD'
-                rules << parsed_content
-              when 'ASM_DATA'
-                data << parsed_content['rules_data'] if parsed_content['rules_data']
-              when 'ASM'
-                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
-                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
-                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-                actions.concat(parsed_content['actions']) if parsed_content['actions']
-              end
+              engine.remove_config_at_path(change.path.to_s)
             end
 
-            if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(
-                telemetry: telemetry,
-                ruleset: Datadog.configuration.appsec.ruleset
-              )
-
-              raise NoRulesError, 'no default rules available' unless settings_rules
-
-              rules = [settings_rules]
+            changes.each do |change|
+              content = repository[change.path]
+              next unless content
+
+              case change.type
+              when :insert, :update
+                # @type var content: Core::Remote::Configuration::Content
+                engine.add_or_update_config(parse_content(content), path: change.path.to_s)
+                content.applied
+              end
             end
 
-            ruleset = AppSec::Processor::RuleMerger.merge(
-              rules: rules,
-              data: data,
-              actions: actions,
-              overrides: overrides,
-              exclusions: exclusions,
-              custom_rules: custom_rules,
-              telemetry: telemetry
-            )
-
-            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
-
-            repository.contents.each do |content|
-              content.applied if ASM_PRODUCTS.include?(content.path.product)
-            end
+            # This is subject to change - we need to remove the reconfiguration mutex
+            # and track usages of each WAF handle instead, so that we know when an old
+            # WAF handle can be finalized.
+            AppSec.reconfigure!
           end
-          # rubocop:enable Metrics/BlockLength
 
           [receiver]
         end
-        # rubocop:enable Metrics/MethodLength
-        # rubocop:enable Metrics/CyclomaticComplexity
 
         private
 
@@ -133,13 +114,7 @@ module Datadog
         end
 
         def parse_content(content)
-          data = content.data.read
-
-          content.data.rewind
-
-          raise ReadError, 'EOF reached' if data.nil?
-
-          JSON.parse(data)
+          JSON.parse(content.data)
         end
       end
     end

data/lib/datadog/appsec/response.rb

@@ -7,6 +7,8 @@ module Datadog
   module AppSec
     # AppSec response
     class Response
+      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
+
       attr_reader :status, :headers, :body
 
       def initialize(status:, headers: {}, body: [])
@@ -37,16 +39,26 @@ module Datadog
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
             headers: {'Content-Type' => content_type},
-            body: [content(content_type)],
+            body: [
+              content(
+                security_response_id: interrupt_params['security_response_id'],
+                content_type: content_type
+              )
+            ],
           )
         end
 
         def redirect_response(interrupt_params)
           status_code = interrupt_params['status_code'].to_i
+          location = interrupt_params.fetch('location')
+
+          if (security_response_id = interrupt_params.fetch('security_response_id'))
+            location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
+          end
 
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => interrupt_params.fetch('location')},
+            headers: {'Location' => location},
             body: [],
           )
         end
@@ -82,16 +94,18 @@ module Datadog
           DEFAULT_CONTENT_TYPE
         end
 
-        def content(content_type)
+        def content(security_response_id:, content_type:)
           content_format = CONTENT_TYPE_TO_FORMAT[content_type]
 
           using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
 
-          if using_default
+          template = if using_default
             Datadog::AppSec::Assets.blocked(format: content_format)
           else
             Datadog.configuration.appsec.block.templates.send(content_format)
           end
+
+          template.gsub(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id.to_s)
         end
       end
     end

data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative 'route_text'
+require_relative 'route_pattern'
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes Rails route patterns into route format, inspired by
+      # OpenAPI v3 path templating
+      #
+      # NOTE: Uses the parsed Journey AST when available
+      #
+      # @api private
+      class RailsRoutePattern
+        DOT_CHAR = '.'
+        GROUP_OPEN_CHAR = '('
+        OPTIONAL_GROUP_PATTERN = /\(([^()]*)\)/
+        NAMED_PARAM_PREFIX_CHAR = ':'
+        GLOB_PARAM_PREFIX_CHAR = '*'
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path_params:, path:)
+          @path_params = path_params
+          @path = path
+
+          if @pattern.is_a?(String)
+            # NOTE: Journey groups without route params are never kept
+            #       Example: `/foo(/bar)` with request `/foo/bar` normalizes to `/foo`
+            pattern = remove_paramless_optional_groups(@pattern)
+            return RoutePattern.new(pattern).normalize(path: path)
+          end
+
+          route_path = @pattern.path
+          route_spec = route_path.spec
+
+          unless route_spec.respond_to?(:type)
+            return RoutePattern.new(route_spec.to_s).normalize(path: path)
+          end
+
+          if route_path.names.empty?
+            route_string = route_spec.to_s
+            return RouteText.escape(route_string) unless route_string.include?(GROUP_OPEN_CHAR)
+          end
+
+          buffer = Buffer.new
+          visit_route_node(route_spec, buffer)
+          buffer.to_path
+        end
+
+        private
+
+        def remove_paramless_optional_groups(pattern)
+          result = pattern
+
+          loop do
+            substituted = result.gsub(OPTIONAL_GROUP_PATTERN) do
+              # NOTE: OPTIONAL_GROUP_PATTERN always captures a string for each gsub match
+              group = ::Regexp.last_match(1) # : String
+              optional_group_has_route_params?(group) ? "(#{group})" : ''
+            end
+
+            return result if substituted == result
+
+            result = substituted
+          end
+        end
+
+        def visit_route_node(node, buffer)
+          case node.type
+          when :CAT
+            visit_route_node(node.left, buffer)
+            visit_route_node(node.right, buffer)
+          when :SLASH
+            buffer.flush
+          when :LITERAL
+            buffer.add_literal(node.left)
+          when :DOT
+            buffer.add_literal(DOT_CHAR)
+          when :SYMBOL, :STAR
+            buffer.add_param(node.name)
+          when :GROUP
+            visit_route_node(node.left, buffer) if group_present?(node.left)
+          end
+        end
+
+        def group_present?(node)
+          param_names = collect_group_param_names(node, [])
+          return false if param_names.empty?
+
+          param_names.all? { |name| param_matched_path?(name.to_sym) }
+        end
+
+        def param_matched_path?(name)
+          return @path_params[name].is_a?(String) unless name == :format
+
+          format = @path_params[:format]
+          return false if !format.is_a?(String) || format.empty?
+
+          dot_index = @path.length - format.length - 1
+          return false if dot_index < 0 || @path[dot_index] != DOT_CHAR
+
+          @path.end_with?(format)
+        end
+
+        def collect_group_param_names(node, memo)
+          case node.type
+          when :SYMBOL, :STAR
+            memo << node.name
+          when :CAT
+            collect_group_param_names(node.left, memo)
+            collect_group_param_names(node.right, memo)
+          end
+
+          memo
+        end
+
+        def optional_group_has_route_params?(group)
+          group.include?(NAMED_PARAM_PREFIX_CHAR) || group.include?(GLOB_PARAM_PREFIX_CHAR)
+        end
+
+        class Buffer
+          def initialize
+            @segments = []
+            @text = +''
+            @params = []
+            @nameless_param_count = 0
+          end
+
+          def add_literal(text)
+            @text << text
+          end
+
+          def add_param(name)
+            @params << name
+          end
+
+          def flush
+            return if @text.empty? && @params.empty?
+
+            @segments << if @params.empty?
+              RouteText.escape(@text)
+            else
+              render_params(@params)
+            end
+
+            @text = +''
+            @params.clear
+          end
+
+          def to_path
+            flush
+
+            "/#{@segments.join('/')}"
+          end
+
+          private
+
+          def render_params(params)
+            names = params.map do |name| # $ String
+              next name unless name.empty?
+
+              @nameless_param_count += 1
+              "param#{@nameless_param_count}"
+            end
+
+            "{#{names.join('+')}}"
+          end
+        end
+      end
+    end
+  end
+end