datadog 2.30.0 → 2.43.0

data/lib/datadog/appsec/metrics/exporter.rb

@@ -3,37 +3,43 @@
 module Datadog
   module AppSec
     module Metrics
-      # A class responsible for exporting WAF and RASP call metrics.
+      # A class responsible for exporting WAF and RASP call metrics
       module Exporter
         module_function
 
         def export_waf_metrics(metrics, span)
           return if metrics.evals.zero?
 
-          span.set_tag('_dd.appsec.waf.timeouts', metrics.timeouts)
-          span.set_tag('_dd.appsec.waf.duration', convert_ns_to_us(metrics.duration_ns))
-          span.set_tag('_dd.appsec.waf.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+          span.set_tag("_dd.appsec.waf.timeouts", metrics.timeouts)
+          span.set_tag("_dd.appsec.waf.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.waf.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
         end
 
         def export_rasp_metrics(metrics, span)
           return if metrics.evals.zero?
 
-          span.set_tag('_dd.appsec.rasp.rule.eval', metrics.evals)
-          span.set_tag('_dd.appsec.rasp.timeout', 1) unless metrics.timeouts.zero?
-          span.set_tag('_dd.appsec.rasp.duration', convert_ns_to_us(metrics.duration_ns))
-          span.set_tag('_dd.appsec.rasp.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+          span.set_tag("_dd.appsec.rasp.rule.eval", metrics.evals)
+          span.set_tag("_dd.appsec.rasp.timeout", 1) unless metrics.timeouts.zero?
+          span.set_tag("_dd.appsec.rasp.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.rasp.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
 
           # NOTE: In case of downstream requests being analyzed additionally
           #       with `Context.run_waf` method, we would need to share it
           #       between two exporting methods
           unless metrics.downstream_requests.zero?
-            span.set_tag('_dd.appsec.downstream_request', metrics.downstream_requests)
+            span.set_tag("_dd.appsec.downstream_request", metrics.downstream_requests)
           end
         end
 
-        # private
+        def export_downstream_response_metrics(metrics, span)
+          metrics.each_pair do |reason, count|
+            next if count.zero?
 
-        def convert_ns_to_us(value)
+            span.set_tag("_dd.appsec.downstream_request.response_body_ignored.#{reason}", count)
+          end
+        end
+
+        private_class_method def convert_ns_to_us(value)
           value / 1000.0
         end
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -5,6 +5,11 @@ module Datadog
     module Metrics
       # A class responsible for reporting WAF and RASP telemetry metrics.
       module Telemetry
+        ACTION_BLOCK = "block_request"
+        ACTION_REDIRECT = "redirect_request"
+        BLOCK_SUCCESS = "success"
+        BLOCK_IRRELEVANT = "irrelevant"
+
         module_function
 
         def report_rasp(type, result, phase: nil)
@@ -18,9 +23,16 @@ module Datadog
 
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
-          AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)
-          AppSec.telemetry.inc(namespace, 'rasp.rule.match', 1, tags: tags) if result.match?
-          AppSec.telemetry.inc(namespace, 'rasp.timeout', 1, tags: tags) if result.timeout?
+          AppSec.telemetry.inc(namespace, "rasp.rule.eval", 1, tags: tags)
+          AppSec.telemetry.inc(namespace, "rasp.timeout", 1, tags: tags) if result.timeout?
+
+          if result.match?
+            blocked = result.actions.key?(ACTION_BLOCK) || result.actions.key?(ACTION_REDIRECT)
+            # NOTE: Mutates tags to avoid an extra hash allocation. Keep this the last .inc call.
+            tags[:block] = blocked ? BLOCK_SUCCESS : BLOCK_IRRELEVANT
+
+            AppSec.telemetry.inc(namespace, "rasp.rule.match", 1, tags: tags)
+          end
         end
       end
     end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -9,7 +9,7 @@ module Datadog
 
         def export_waf_request_metrics(metrics, context)
           AppSec.telemetry.inc(
-            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            Ext::TELEMETRY_METRICS_NAMESPACE, "waf.requests", 1,
             tags: {
               waf_version: WAF::VERSION::BASE_STRING,
               event_rules_version: context.waf_runner_ruleset_version,
@@ -17,7 +17,7 @@ module Datadog
               waf_error: metrics.errors.positive?.to_s,
               waf_timeout: metrics.timeouts.positive?.to_s,
               request_blocked: context.interrupted?.to_s,
-              block_failure: 'false',
+              block_failure: "false",
               rate_limited: (!context.trace.sampled?).to_s,
               input_truncated: metrics.inputs_truncated.positive?.to_s,
             }
@@ -30,12 +30,12 @@ module Datadog
 
           if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
             AppSec.telemetry.inc(
-              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, 'api_security.missing_route', 1,
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.missing_route", 1,
               tags: {framework: web_framework}
             )
           end
 
-          metric_name = context.state[:schema_extracted] ? 'schema' : 'no_schema'
+          metric_name = context.state[:schema_extracted] ? "schema" : "no_schema"
           AppSec.telemetry.inc(
             AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
             tags: {framework: web_framework}

data/lib/datadog/appsec/metrics.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'metrics/collector'
-require_relative 'metrics/exporter'
-require_relative 'metrics/telemetry'
-require_relative 'metrics/telemetry_exporter'
+require_relative "metrics/collector"
+require_relative "metrics/exporter"
+require_relative "metrics/telemetry"
+require_relative "metrics/telemetry_exporter"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../event'
-require_relative '../../security_event'
-require_relative '../../instrumentation/gateway'
+require_relative "../../event"
+require_relative "../../security_event"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -10,9 +10,9 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
-          ARBITRARY_VALUE = 'invalid'
-          EVENT_LOGIN_SUCCESS = 'users.login.success'
-          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          ARBITRARY_VALUE = "invalid"
+          EVENT_LOGIN_SUCCESS = "users.login.success"
+          EVENT_LOGIN_FAILURE = "users.login.failure"
           WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
 
           class << self
@@ -24,18 +24,21 @@ module Datadog
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user') do |stack, user|
+              gateway.watch("identity.set_user") do |stack, user|
                 context = AppSec.active_context
+                next stack.call(user) unless context
+
+                context.state[:has_identity_event] = true
 
                 if user.id.nil? && user.login.nil? && user.session_id.nil?
-                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  Datadog.logger.debug { "AppSec: skipping WAF check because no user information was provided" }
                   next stack.call(user)
                 end
 
                 persistent_data = {}
-                persistent_data['usr.id'] = user.id if user.id
-                persistent_data['usr.login'] = user.login if user.login
-                persistent_data['usr.session_id'] = user.session_id if user.session_id
+                persistent_data["usr.id"] = user.id if user.id
+                persistent_data["usr.login"] = user.login if user.login
+                persistent_data["usr.session_id"] = user.session_id if user.session_id
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
@@ -55,8 +58,11 @@ module Datadog
             end
 
             def watch_user_login(gateway = Instrumentation.gateway)
-              gateway.watch('appsec.events.user_lifecycle') do |stack, kind|
+              gateway.watch("appsec.events.user_lifecycle") do |stack, kind|
                 context = AppSec.active_context
+                next stack.call(kind) unless context
+
+                context.state[:has_identity_event] = true
 
                 next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
 

data/lib/datadog/appsec/monitor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'monitor/gateway/watcher'
+require_relative "monitor/gateway/watcher"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../assets'
+require_relative "../assets"
 
 module Datadog
   module AppSec
@@ -15,15 +15,15 @@ module Datadog
               JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
             when :risky
               Datadog.logger.warn(
-                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
-                'The `:recommended` ruleset will be used instead.' \
-                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                "The :risky Application Security Management ruleset has been deprecated and no longer available." \
+                "The `:recommended` ruleset will be used instead." \
+                "Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block."
               )
               JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
             when String
               JSON.parse(File.read(File.expand_path(ruleset)))
             when File, StringIO
-              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              JSON.parse(ruleset.read || "").tap { ruleset.rewind }
             when Hash
               ruleset
             else
@@ -34,15 +34,15 @@ module Datadog
               "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
             end
 
-            telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: "libddwaf ruleset failed to load")
 
             raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
-            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
+            data << denylist_data("blocked_ips", ip_denylist) if ip_denylist.any?
+            data << denylist_data("blocked_users", user_id_denylist) if user_id_denylist.any?
 
             data
           end
@@ -57,9 +57,9 @@ module Datadog
 
           def denylist_data(id, denylist)
             {
-              'id' => id,
-              'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
+              "id" => id,
+              "type" => "data_with_expiration",
+              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} },
             }
           end
 
@@ -76,38 +76,38 @@ module Datadog
             exclusions = []
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    'list' => pass
-                  }
-                }
+                    "list" => pass,
+                  },
+                },
               ],
-              'id' => SecureRandom.uuid,
+              "id" => SecureRandom.uuid,
             }
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    'list' => monitor
-                  }
-                }
+                    "list" => monitor,
+                  },
+                },
               ],
-              'id' => SecureRandom.uuid,
-              'on_match' => 'monitor'
+              "id" => SecureRandom.uuid,
+              "on_match" => "monitor",
             }
 
             exclusions

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../core/rate_limiter'
+require_relative "../core/rate_limiter"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/remote.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_loader'
+require_relative "../core/remote/dispatcher"
+require_relative "processor/rule_loader"
 
 module Datadog
   module AppSec
@@ -56,10 +56,10 @@ module Datadog
         ].freeze
 
         ASM_PRODUCTS = [
-          'ASM_DD',       # Datadog employee issued configuration
-          'ASM',          # customer issued configuration (rulesets, passlist...)
-          'ASM_FEATURES', # capabilities
-          'ASM_DATA',     # config files (IP addresses or users for blocking)
+          "ASM_DD",       # Datadog employee issued configuration
+          "ASM",          # customer issued configuration (rulesets, passlist...)
+          "ASM_FEATURES", # capabilities
+          "ASM_DATA",     # config files (IP addresses or users for blocking)
         ].freeze
 
         def capabilities
@@ -78,17 +78,23 @@ module Datadog
             engine = AppSec.security_engine
             next unless engine
 
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
+            changes.each do |change|
+              next unless change.type == :delete
+
+              engine.remove_config_at_path(change.path.to_s)
+            end
+
             changes.each do |change|
               content = repository[change.path]
-              next unless content || change.type == :delete
+              next unless content
 
               case change.type
               when :insert, :update
                 # @type var content: Core::Remote::Configuration::Content
                 engine.add_or_update_config(parse_content(content), path: change.path.to_s)
                 content.applied
-              when :delete
-                engine.remove_config_at_path(change.path.to_s)
               end
             end
 

data/lib/datadog/appsec/response.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative 'assets'
-require_relative 'utils/http/media_range'
+require_relative "assets"
+require_relative "utils/http/media_range"
 
 module Datadog
   module AppSec
     # AppSec response
     class Response
-      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
+      SECURITY_RESPONSE_ID_PLACEHOLDER = "[security_response_id]"
 
       attr_reader :status, :headers, :body
 
@@ -23,7 +23,7 @@ module Datadog
 
       class << self
         def from_interrupt_params(interrupt_params, http_accept_header)
-          return redirect_response(interrupt_params) if interrupt_params['location']
+          return redirect_response(interrupt_params) if interrupt_params["location"]
 
           block_response(interrupt_params, http_accept_header)
         end
@@ -31,55 +31,55 @@ module Datadog
         private
 
         def block_response(interrupt_params, http_accept_header)
-          content_type = case interrupt_params['type']
-          when nil, 'auto' then content_type(http_accept_header)
-          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          content_type = case interrupt_params["type"]
+          when nil, "auto" then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params["type"], DEFAULT_CONTENT_TYPE)
           end
 
           Response.new(
-            status: interrupt_params['status_code']&.to_i || 403,
-            headers: {'Content-Type' => content_type},
+            status: interrupt_params["status_code"]&.to_i || 403,
+            headers: {"Content-Type" => content_type},
             body: [
               content(
-                security_response_id: interrupt_params['security_response_id'],
+                security_response_id: interrupt_params["security_response_id"],
                 content_type: content_type
-              )
+              ),
             ],
           )
         end
 
         def redirect_response(interrupt_params)
-          status_code = interrupt_params['status_code'].to_i
-          location = interrupt_params.fetch('location')
+          status_code = interrupt_params["status_code"].to_i
+          location = interrupt_params.fetch("location")
 
-          if (security_response_id = interrupt_params.fetch('security_response_id'))
+          if (security_response_id = interrupt_params.fetch("security_response_id"))
             location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
           end
 
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => location},
+            headers: {"Location" => location},
             body: [],
           )
         end
 
         CONTENT_TYPE_TO_FORMAT = {
-          'application/json' => :json,
-          'text/html' => :html,
-          'text/plain' => :text,
+          "application/json" => :json,
+          "text/html" => :html,
+          "text/plain" => :text,
         }.freeze
 
         FORMAT_TO_CONTENT_TYPE = {
-          'json' => 'application/json',
-          'html' => 'text/html',
+          "json" => "application/json",
+          "html" => "text/html",
         }.freeze
 
-        DEFAULT_CONTENT_TYPE = 'application/json'
+        DEFAULT_CONTENT_TYPE = "application/json"
 
         def content_type(http_accept_header)
           return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = http_accept_header.split(',').map(&:strip)
+          accept_types = http_accept_header.split(",").map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
 

data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative "route_text"
+require_relative "route_pattern"
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes Rails route patterns into route format, inspired by
+      # OpenAPI v3 path templating
+      #
+      # NOTE: Uses the parsed Journey AST when available
+      #
+      # @api private
+      class RailsRoutePattern
+        DOT_CHAR = "."
+        GROUP_OPEN_CHAR = "("
+        OPTIONAL_GROUP_PATTERN = /\(([^()]*)\)/
+        NAMED_PARAM_PREFIX_CHAR = ":"
+        GLOB_PARAM_PREFIX_CHAR = "*"
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path_params:, path:)
+          @path_params = path_params
+          @path = path
+
+          if @pattern.is_a?(String)
+            # NOTE: Journey groups without route params are never kept
+            #       Example: `/foo(/bar)` with request `/foo/bar` normalizes to `/foo`
+            pattern = remove_paramless_optional_groups(@pattern)
+            return RoutePattern.new(pattern).normalize(path: path)
+          end
+
+          route_path = @pattern.path
+          route_spec = route_path.spec
+
+          unless route_spec.respond_to?(:type)
+            return RoutePattern.new(route_spec.to_s).normalize(path: path)
+          end
+
+          if route_path.names.empty?
+            route_string = route_spec.to_s
+            return RouteText.escape(route_string) unless route_string.include?(GROUP_OPEN_CHAR)
+          end
+
+          buffer = Buffer.new
+          visit_route_node(route_spec, buffer)
+          buffer.to_path
+        end
+
+        private
+
+        def remove_paramless_optional_groups(pattern)
+          result = pattern
+
+          loop do
+            substituted = result.gsub(OPTIONAL_GROUP_PATTERN) do
+              # NOTE: OPTIONAL_GROUP_PATTERN always captures a string for each gsub match
+              group = ::Regexp.last_match(1) # : String
+              optional_group_has_route_params?(group) ? "(#{group})" : ""
+            end
+
+            return result if substituted == result
+
+            result = substituted
+          end
+        end
+
+        def visit_route_node(node, buffer)
+          case node.type
+          when :CAT
+            visit_route_node(node.left, buffer)
+            visit_route_node(node.right, buffer)
+          when :SLASH
+            buffer.flush
+          when :LITERAL
+            buffer.add_literal(node.left)
+          when :DOT
+            buffer.add_literal(DOT_CHAR)
+          when :SYMBOL, :STAR
+            buffer.add_param(node.name)
+          when :GROUP
+            visit_route_node(node.left, buffer) if group_present?(node.left)
+          end
+        end
+
+        def group_present?(node)
+          param_names = collect_group_param_names(node, [])
+          return false if param_names.empty?
+
+          param_names.all? { |name| param_matched_path?(name.to_sym) }
+        end
+
+        def param_matched_path?(name)
+          return @path_params[name].is_a?(String) unless name == :format
+
+          format = @path_params[:format]
+          return false if !format.is_a?(String) || format.empty?
+
+          dot_index = @path.length - format.length - 1
+          return false if dot_index < 0 || @path[dot_index] != DOT_CHAR
+
+          @path.end_with?(format)
+        end
+
+        def collect_group_param_names(node, memo)
+          case node.type
+          when :SYMBOL, :STAR
+            memo << node.name
+          when :CAT
+            collect_group_param_names(node.left, memo)
+            collect_group_param_names(node.right, memo)
+          end
+
+          memo
+        end
+
+        def optional_group_has_route_params?(group)
+          group.include?(NAMED_PARAM_PREFIX_CHAR) || group.include?(GLOB_PARAM_PREFIX_CHAR)
+        end
+
+        class Buffer
+          def initialize
+            @segments = []
+            @text = +""
+            @params = []
+            @nameless_param_count = 0
+          end
+
+          def add_literal(text)
+            @text << text
+          end
+
+          def add_param(name)
+            @params << name
+          end
+
+          def flush
+            return if @text.empty? && @params.empty?
+
+            @segments << if @params.empty?
+              RouteText.escape(@text)
+            else
+              render_params(@params)
+            end
+
+            @text = +""
+            @params.clear
+          end
+
+          def to_path
+            flush
+
+            "/#{@segments.join("/")}"
+          end
+
+          private
+
+          def render_params(params)
+            names = params.map do |name| # $ String
+              next name unless name.empty?
+
+              @nameless_param_count += 1
+              "param#{@nameless_param_count}"
+            end
+
+            "{#{names.join("+")}}"
+          end
+        end
+      end
+    end
+  end
+end