datadog 2.28.0 → 2.41.0

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'gateway/middleware'
+require_relative "gateway/middleware"
 
 module Datadog
   module AppSec
@@ -10,18 +10,9 @@ module Datadog
       class Gateway
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
-          @pushed_events = {}
         end
 
-        # NOTE: Be careful with pushed names because every pushed event name
-        #       is recorded in order to provide an ability to any subscriber
-        #       to check wether an arbitrary event had happened.
-        #
-        # WARNING: If we start pushing generated names we should consider
-        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          @pushed_events[name] = true
-
           block ||= -> {}
           middlewares_for_name = @middlewares[name]
 
@@ -41,12 +32,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
-        end
-
-        def pushed?(name)
-          @pushed_events.key?(name)
+        def watch(name, &block)
+          @middlewares[name] << Middleware.new(&block)
         end
       end
 

data/lib/datadog/appsec/metrics/collector.rb

@@ -17,7 +17,15 @@ module Datadog
           keyword_init: true
         )
 
-        attr_reader :waf, :rasp
+        DownstreamResponseStore = Struct.new(
+          :content_type_invalid,
+          :content_length_missing,
+          :content_length_too_big,
+          :content_exceed_content_length,
+          keyword_init: true
+        )
+
+        attr_reader :waf, :rasp, :downstream_responses
 
         def initialize
           @mutex = Mutex.new
@@ -31,6 +39,11 @@ module Datadog
             evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
             duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
           )
+
+          @downstream_responses = DownstreamResponseStore.new(
+            content_type_invalid: 0, content_length_missing: 0,
+            content_length_too_big: 0, content_exceed_content_length: 0
+          )
         end
 
         def record_waf(result)
@@ -57,6 +70,10 @@ module Datadog
             @rasp.downstream_requests += 1 if type == Ext::RASP_SSRF && phase == Ext::RASP_REQUEST_PHASE
           end
         end
+
+        def record_ignored_downstream_response_body(reason)
+          @mutex.synchronize { @downstream_responses[reason] += 1 }
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics/exporter.rb

@@ -3,37 +3,43 @@
 module Datadog
   module AppSec
     module Metrics
-      # A class responsible for exporting WAF and RASP call metrics.
+      # A class responsible for exporting WAF and RASP call metrics
       module Exporter
         module_function
 
         def export_waf_metrics(metrics, span)
           return if metrics.evals.zero?
 
-          span.set_tag('_dd.appsec.waf.timeouts', metrics.timeouts)
-          span.set_tag('_dd.appsec.waf.duration', convert_ns_to_us(metrics.duration_ns))
-          span.set_tag('_dd.appsec.waf.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+          span.set_tag("_dd.appsec.waf.timeouts", metrics.timeouts)
+          span.set_tag("_dd.appsec.waf.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.waf.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
         end
 
         def export_rasp_metrics(metrics, span)
           return if metrics.evals.zero?
 
-          span.set_tag('_dd.appsec.rasp.rule.eval', metrics.evals)
-          span.set_tag('_dd.appsec.rasp.timeout', 1) unless metrics.timeouts.zero?
-          span.set_tag('_dd.appsec.rasp.duration', convert_ns_to_us(metrics.duration_ns))
-          span.set_tag('_dd.appsec.rasp.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+          span.set_tag("_dd.appsec.rasp.rule.eval", metrics.evals)
+          span.set_tag("_dd.appsec.rasp.timeout", 1) unless metrics.timeouts.zero?
+          span.set_tag("_dd.appsec.rasp.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.rasp.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
 
           # NOTE: In case of downstream requests being analyzed additionally
           #       with `Context.run_waf` method, we would need to share it
           #       between two exporting methods
           unless metrics.downstream_requests.zero?
-            span.set_tag('_dd.appsec.downstream_request', metrics.downstream_requests)
+            span.set_tag("_dd.appsec.downstream_request", metrics.downstream_requests)
           end
         end
 
-        # private
+        def export_downstream_response_metrics(metrics, span)
+          metrics.each_pair do |reason, count|
+            next if count.zero?
 
-        def convert_ns_to_us(value)
+            span.set_tag("_dd.appsec.downstream_request.response_body_ignored.#{reason}", count)
+          end
+        end
+
+        private_class_method def convert_ns_to_us(value)
           value / 1000.0
         end
       end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -5,6 +5,11 @@ module Datadog
     module Metrics
       # A class responsible for reporting WAF and RASP telemetry metrics.
       module Telemetry
+        ACTION_BLOCK = "block_request"
+        ACTION_REDIRECT = "redirect_request"
+        BLOCK_SUCCESS = "success"
+        BLOCK_IRRELEVANT = "irrelevant"
+
         module_function
 
         def report_rasp(type, result, phase: nil)
@@ -18,9 +23,16 @@ module Datadog
 
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE
 
-          AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)
-          AppSec.telemetry.inc(namespace, 'rasp.rule.match', 1, tags: tags) if result.match?
-          AppSec.telemetry.inc(namespace, 'rasp.timeout', 1, tags: tags) if result.timeout?
+          AppSec.telemetry.inc(namespace, "rasp.rule.eval", 1, tags: tags)
+          AppSec.telemetry.inc(namespace, "rasp.timeout", 1, tags: tags) if result.timeout?
+
+          if result.match?
+            blocked = result.actions.key?(ACTION_BLOCK) || result.actions.key?(ACTION_REDIRECT)
+            # NOTE: Mutates tags to avoid an extra hash allocation. Keep this the last .inc call.
+            tags[:block] = blocked ? BLOCK_SUCCESS : BLOCK_IRRELEVANT
+
+            AppSec.telemetry.inc(namespace, "rasp.rule.match", 1, tags: tags)
+          end
         end
       end
     end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -9,7 +9,7 @@ module Datadog
 
         def export_waf_request_metrics(metrics, context)
           AppSec.telemetry.inc(
-            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            Ext::TELEMETRY_METRICS_NAMESPACE, "waf.requests", 1,
             tags: {
               waf_version: WAF::VERSION::BASE_STRING,
               event_rules_version: context.waf_runner_ruleset_version,
@@ -17,12 +17,30 @@ module Datadog
               waf_error: metrics.errors.positive?.to_s,
               waf_timeout: metrics.timeouts.positive?.to_s,
               request_blocked: context.interrupted?.to_s,
-              block_failure: 'false',
+              block_failure: "false",
               rate_limited: (!context.trace.sampled?).to_s,
               input_truncated: metrics.inputs_truncated.positive?.to_s,
             }
           )
         end
+
+        def export_api_security_metrics(context)
+          web_framework = context.state[:web_framework]
+          return unless web_framework
+
+          if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
+            AppSec.telemetry.inc(
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.missing_route", 1,
+              tags: {framework: web_framework}
+            )
+          end
+
+          metric_name = context.state[:schema_extracted] ? "schema" : "no_schema"
+          AppSec.telemetry.inc(
+            AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
+            tags: {framework: web_framework}
+          )
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'metrics/collector'
-require_relative 'metrics/exporter'
-require_relative 'metrics/telemetry'
-require_relative 'metrics/telemetry_exporter'
+require_relative "metrics/collector"
+require_relative "metrics/exporter"
+require_relative "metrics/telemetry"
+require_relative "metrics/telemetry_exporter"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../event'
-require_relative '../../security_event'
-require_relative '../../instrumentation/gateway'
+require_relative "../../event"
+require_relative "../../security_event"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -10,9 +10,9 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
-          ARBITRARY_VALUE = 'invalid'
-          EVENT_LOGIN_SUCCESS = 'users.login.success'
-          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          ARBITRARY_VALUE = "invalid"
+          EVENT_LOGIN_SUCCESS = "users.login.success"
+          EVENT_LOGIN_FAILURE = "users.login.failure"
           WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
 
           class << self
@@ -24,18 +24,21 @@ module Datadog
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user', :appsec) do |stack, user|
+              gateway.watch("identity.set_user") do |stack, user|
                 context = AppSec.active_context
+                next stack.call(user) unless context
+
+                context.state[:has_identity_event] = true
 
                 if user.id.nil? && user.login.nil? && user.session_id.nil?
-                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  Datadog.logger.debug { "AppSec: skipping WAF check because no user information was provided" }
                   next stack.call(user)
                 end
 
                 persistent_data = {}
-                persistent_data['usr.id'] = user.id if user.id
-                persistent_data['usr.login'] = user.login if user.login
-                persistent_data['usr.session_id'] = user.session_id if user.session_id
+                persistent_data["usr.id"] = user.id if user.id
+                persistent_data["usr.login"] = user.login if user.login
+                persistent_data["usr.session_id"] = user.session_id if user.session_id
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
@@ -55,8 +58,11 @@ module Datadog
             end
 
             def watch_user_login(gateway = Instrumentation.gateway)
-              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+              gateway.watch("appsec.events.user_lifecycle") do |stack, kind|
                 context = AppSec.active_context
+                next stack.call(kind) unless context
+
+                context.state[:has_identity_event] = true
 
                 next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
 

data/lib/datadog/appsec/monitor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'monitor/gateway/watcher'
+require_relative "monitor/gateway/watcher"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../assets'
+require_relative "../assets"
 
 module Datadog
   module AppSec
@@ -15,15 +15,15 @@ module Datadog
               JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
             when :risky
               Datadog.logger.warn(
-                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
-                'The `:recommended` ruleset will be used instead.' \
-                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                "The :risky Application Security Management ruleset has been deprecated and no longer available." \
+                "The `:recommended` ruleset will be used instead." \
+                "Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block."
               )
               JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
             when String
               JSON.parse(File.read(File.expand_path(ruleset)))
             when File, StringIO
-              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              JSON.parse(ruleset.read || "").tap { ruleset.rewind }
             when Hash
               ruleset
             else
@@ -34,15 +34,15 @@ module Datadog
               "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
             end
 
-            telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: "libddwaf ruleset failed to load")
 
             raise e
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
             data = []
-            data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
-            data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
+            data << denylist_data("blocked_ips", ip_denylist) if ip_denylist.any?
+            data << denylist_data("blocked_users", user_id_denylist) if user_id_denylist.any?
 
             data
           end
@@ -57,9 +57,9 @@ module Datadog
 
           def denylist_data(id, denylist)
             {
-              'id' => id,
-              'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
+              "id" => id,
+              "type" => "data_with_expiration",
+              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} },
             }
           end
 
@@ -76,38 +76,38 @@ module Datadog
             exclusions = []
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    'list' => pass
-                  }
-                }
+                    "list" => pass,
+                  },
+                },
               ],
-              'id' => SecureRandom.uuid,
+              "id" => SecureRandom.uuid,
             }
 
             exclusions << {
-              'conditions' => [
+              "conditions" => [
                 {
-                  'operator' => 'ip_match',
-                  'parameters' => {
-                    'inputs' => [
+                  "operator" => "ip_match",
+                  "parameters" => {
+                    "inputs" => [
                       {
-                        'address' => 'http.client_ip'
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    'list' => monitor
-                  }
-                }
+                    "list" => monitor,
+                  },
+                },
               ],
-              'id' => SecureRandom.uuid,
-              'on_match' => 'monitor'
+              "id" => SecureRandom.uuid,
+              "on_match" => "monitor",
             }
 
             exclusions

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../core/rate_limiter'
+require_relative "../core/rate_limiter"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/remote.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../core/remote/dispatcher'
-require_relative 'processor/rule_loader'
+require_relative "../core/remote/dispatcher"
+require_relative "processor/rule_loader"
 
 module Datadog
   module AppSec
@@ -56,10 +56,10 @@ module Datadog
         ].freeze
 
         ASM_PRODUCTS = [
-          'ASM_DD',       # Datadog employee issued configuration
-          'ASM',          # customer issued configuration (rulesets, passlist...)
-          'ASM_FEATURES', # capabilities
-          'ASM_DATA',     # config files (IP addresses or users for blocking)
+          "ASM_DD",       # Datadog employee issued configuration
+          "ASM",          # customer issued configuration (rulesets, passlist...)
+          "ASM_FEATURES", # capabilities
+          "ASM_DATA",     # config files (IP addresses or users for blocking)
         ].freeze
 
         def capabilities
@@ -78,17 +78,23 @@ module Datadog
             engine = AppSec.security_engine
             next unless engine
 
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
+            changes.each do |change|
+              next unless change.type == :delete
+
+              engine.remove_config_at_path(change.path.to_s)
+            end
+
             changes.each do |change|
               content = repository[change.path]
-              next unless content || change.type == :delete
+              next unless content
 
               case change.type
               when :insert, :update
                 # @type var content: Core::Remote::Configuration::Content
                 engine.add_or_update_config(parse_content(content), path: change.path.to_s)
                 content.applied
-              when :delete
-                engine.remove_config_at_path(change.path.to_s)
               end
             end
 

data/lib/datadog/appsec/response.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative 'assets'
-require_relative 'utils/http/media_range'
+require_relative "assets"
+require_relative "utils/http/media_range"
 
 module Datadog
   module AppSec
     # AppSec response
     class Response
-      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
+      SECURITY_RESPONSE_ID_PLACEHOLDER = "[security_response_id]"
 
       attr_reader :status, :headers, :body
 
@@ -23,7 +23,7 @@ module Datadog
 
       class << self
         def from_interrupt_params(interrupt_params, http_accept_header)
-          return redirect_response(interrupt_params) if interrupt_params['location']
+          return redirect_response(interrupt_params) if interrupt_params["location"]
 
           block_response(interrupt_params, http_accept_header)
         end
@@ -31,55 +31,55 @@ module Datadog
         private
 
         def block_response(interrupt_params, http_accept_header)
-          content_type = case interrupt_params['type']
-          when nil, 'auto' then content_type(http_accept_header)
-          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          content_type = case interrupt_params["type"]
+          when nil, "auto" then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params["type"], DEFAULT_CONTENT_TYPE)
           end
 
           Response.new(
-            status: interrupt_params['status_code']&.to_i || 403,
-            headers: {'Content-Type' => content_type},
+            status: interrupt_params["status_code"]&.to_i || 403,
+            headers: {"Content-Type" => content_type},
             body: [
               content(
-                security_response_id: interrupt_params['security_response_id'],
+                security_response_id: interrupt_params["security_response_id"],
                 content_type: content_type
-              )
+              ),
             ],
           )
         end
 
         def redirect_response(interrupt_params)
-          status_code = interrupt_params['status_code'].to_i
-          location = interrupt_params.fetch('location')
+          status_code = interrupt_params["status_code"].to_i
+          location = interrupt_params.fetch("location")
 
-          if (security_response_id = interrupt_params.fetch('security_response_id'))
+          if (security_response_id = interrupt_params.fetch("security_response_id"))
             location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
           end
 
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => location},
+            headers: {"Location" => location},
             body: [],
           )
         end
 
         CONTENT_TYPE_TO_FORMAT = {
-          'application/json' => :json,
-          'text/html' => :html,
-          'text/plain' => :text,
+          "application/json" => :json,
+          "text/html" => :html,
+          "text/plain" => :text,
         }.freeze
 
         FORMAT_TO_CONTENT_TYPE = {
-          'json' => 'application/json',
-          'html' => 'text/html',
+          "json" => "application/json",
+          "html" => "text/html",
         }.freeze
 
-        DEFAULT_CONTENT_TYPE = 'application/json'
+        DEFAULT_CONTENT_TYPE = "application/json"
 
         def content_type(http_accept_header)
           return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = http_accept_header.split(',').map(&:strip)
+          accept_types = http_accept_header.split(",").map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!