datadog 2.30.0 → 2.45.0

data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../ext'
-require_relative '../configuration'
-require_relative '../data_extractor'
-require_relative '../../../trace_keeper'
+require_relative "../ext"
+require_relative "../configuration"
+require_relative "../data_extractor"
+require_relative "../../../trace_keeper"
 
 module Datadog
   module AppSec
@@ -18,11 +18,12 @@ module Datadog
               return result unless AppSec.enabled?
               return result if @_datadog_appsec_skip_track_login_event
               return result unless Configuration.auto_user_instrumentation_enabled?
-              return result unless AppSec.active_context
 
               context = AppSec.active_context
+              return result unless context
+
               if context.trace.nil? || context.span.nil?
-                Datadog.logger.debug { 'AppSec: unable to track signin events, due to missing trace or span' }
+                Datadog.logger.debug { "AppSec: unable to track signin events, due to missing trace or span" }
                 return result
               end
 
@@ -30,13 +31,13 @@ module Datadog
 
               if result
                 record_successful_signin(context, resource)
-                Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_LOGIN_SUCCESS)
+                Instrumentation.gateway.push("appsec.events.user_lifecycle", Ext::EVENT_LOGIN_SUCCESS)
 
                 return result
               end
 
               record_failed_signin(context, resource)
-              Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_LOGIN_FAILURE)
+              Instrumentation.gateway.push("appsec.events.user_lifecycle", Ext::EVENT_LOGIN_FAILURE)
 
               result
             end
@@ -55,7 +56,7 @@ module Datadog
               end
 
               context.span[Ext::TAG_LOGIN_SUCCESS_USR_LOGIN] ||= login
-              context.span[Ext::TAG_LOGIN_SUCCESS_TRACK] = 'true'
+              context.span[Ext::TAG_LOGIN_SUCCESS_TRACK] = "true"
               context.span[Ext::TAG_DD_USR_LOGIN] = login
               context.span[Ext::TAG_DD_LOGIN_SUCCESS_MODE] = Configuration.auto_user_instrumentation_mode
 
@@ -63,14 +64,14 @@ module Datadog
               #       and because of that we will trigger an additional event even
               #       if it was already done via the SDK
               AppSec::Instrumentation.gateway.push(
-                'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, login)
+                "identity.set_user", AppSec::Instrumentation::Gateway::User.new(id, login)
               )
             end
 
             def record_failed_signin(context, resource)
               extractor = DataExtractor.new(mode: Configuration.auto_user_instrumentation_mode)
 
-              context.span[Ext::TAG_LOGIN_FAILURE_TRACK] = 'true'
+              context.span[Ext::TAG_LOGIN_FAILURE_TRACK] = "true"
               context.span[Ext::TAG_DD_LOGIN_FAILURE_MODE] = Configuration.auto_user_instrumentation_mode
 
               unless resource
@@ -78,7 +79,7 @@ module Datadog
 
                 context.span[Ext::TAG_DD_USR_LOGIN] = login
                 context.span[Ext::TAG_LOGIN_FAILURE_USR_LOGIN] ||= login
-                context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= 'false'
+                context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= "false"
 
                 return
               end
@@ -93,7 +94,7 @@ module Datadog
 
               context.span[Ext::TAG_DD_USR_LOGIN] = login
               context.span[Ext::TAG_LOGIN_FAILURE_USR_LOGIN] ||= login
-              context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= 'true'
+              context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= "true"
             end
           end
         end

data/lib/datadog/appsec/contrib/devise/patches/signup_tracking_patch.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../ext'
-require_relative '../configuration'
-require_relative '../data_extractor'
-require_relative '../../../trace_keeper'
+require_relative "../ext"
+require_relative "../configuration"
+require_relative "../data_extractor"
+require_relative "../../../trace_keeper"
 
 module Datadog
   module AppSec
@@ -21,7 +21,7 @@ module Datadog
                 context = AppSec.active_context
 
                 if context.trace.nil? || context.span.nil?
-                  Datadog.logger.debug { 'AppSec: unable to track signup events, due to missing trace or span' }
+                  Datadog.logger.debug { "AppSec: unable to track signup events, due to missing trace or span" }
                   next yield(resource) if block_given?
                 end
 
@@ -29,7 +29,7 @@ module Datadog
 
                 TraceKeeper.keep!(context.trace)
                 record_successful_signup(context, resource)
-                Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_SIGNUP)
+                Instrumentation.gateway.push("appsec.events.user_lifecycle", Ext::EVENT_SIGNUP)
 
                 yield(resource) if block_given?
               end
@@ -43,7 +43,7 @@ module Datadog
               id = extractor.extract_id(resource)
               login = extractor.extract_login(resource_params) || extractor.extract_login(resource)
 
-              context.span[Ext::TAG_SIGNUP_TRACK] = 'true'
+              context.span[Ext::TAG_SIGNUP_TRACK] = "true"
               context.span[Ext::TAG_DD_USR_LOGIN] = login
               context.span[Ext::TAG_SIGNUP_USR_LOGIN] ||= login
               context.span[Ext::TAG_DD_SIGNUP_MODE] = Configuration.auto_user_instrumentation_mode
@@ -59,7 +59,7 @@ module Datadog
               #       and because of that we will trigger an additional event even
               #       if it was already done via the SDK
               AppSec::Instrumentation.gateway.push(
-                'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, login)
+                "identity.set_user", AppSec::Instrumentation::Gateway::User.new(id, login)
               )
             end
           end

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'ext'
-require_relative '../../anonymizer'
+require_relative "ext"
+require_relative "../../anonymizer"
 
 module Datadog
   module AppSec
@@ -9,8 +9,8 @@ module Datadog
       module Devise
         # A Rack middleware capable of tracking currently signed user
         class TrackingMiddleware
-          WARDEN_KEY = 'warden'
-          SESSION_ID_KEY = 'session_id'
+          WARDEN_KEY = "warden"
+          SESSION_ID_KEY = "session_id"
 
           def initialize(app)
             @app = app
@@ -23,13 +23,13 @@ module Datadog
             return @app.call(env) unless AppSec.active_context
 
             unless env.key?(WARDEN_KEY)
-              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing warden manager' }
+              Datadog.logger.debug { "AppSec: unable to track requests, due to missing warden manager" }
               return @app.call(env)
             end
 
             context = AppSec.active_context
             if context.trace.nil? || context.span.nil?
-              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing trace or span' }
+              Datadog.logger.debug { "AppSec: unable to track requests, due to missing trace or span" }
               return @app.call(env)
             end
 
@@ -49,7 +49,7 @@ module Datadog
                 #        passing them at first place.
                 #        This is a temporary situation until we refactor events model.
                 AppSec::Instrumentation.gateway.push(
-                  'identity.set_user', AppSec::Instrumentation::Gateway::User.new(user_id, nil, user_session_id)
+                  "identity.set_user", AppSec::Instrumentation::Gateway::User.new(user_id, nil, user_session_id)
                 )
               end
 
@@ -67,7 +67,7 @@ module Datadog
             session_serializer = warden.session_serializer
 
             key = session_key_for(session_serializer, ::Devise.default_scope)
-            id = session_serializer.session[key]&.dig(0, 0)
+            id = extract_record_id(session_serializer.session[key])
 
             return id if ::Devise.mappings.size == 1
             return "#{::Devise.default_scope}:#{id}" if id
@@ -76,7 +76,7 @@ module Datadog
               next if scope == ::Devise.default_scope
 
               key = session_key_for(session_serializer, scope)
-              id = session_serializer.session[key]&.dig(0, 0)
+              id = extract_record_id(session_serializer.session[key])
 
               return "#{scope}:#{id}" if id
             end
@@ -84,6 +84,21 @@ module Datadog
             nil
           end
 
+          # NOTE: This supports Devise's default session serialization:
+          #       `[record.to_key, record.authenticatable_salt]`,
+          #       e.g. `[[1], "0abc..."]`
+          #
+          # WARNING: Applications can redefine Warden serialization, replacing
+          #          this default format
+          def extract_record_id(serialized_record)
+            return unless serialized_record.is_a?(Array)
+
+            record_key = serialized_record[0]
+            return unless record_key.is_a?(Array)
+
+            record_key[0]
+          end
+
           def session_key_for(session_serializer, scope)
             @devise_session_scope_keys[scope] ||= session_serializer.key_for(scope)
           end

data/lib/datadog/appsec/contrib/excon/integration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
-require_relative 'patcher'
+require_relative "../integration"
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -11,12 +11,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('0.50.0')
+          MINIMUM_VERSION = Gem::Version.new("0.50.0")
 
           register_as :excon
 
           def self.version
-            Gem.loaded_specs['excon']&.version
+            Gem.loaded_specs["excon"]&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/excon/patcher.rb

@@ -17,7 +17,7 @@ module Datadog
           end
 
           def patch
-            require_relative 'ssrf_detection_middleware'
+            require_relative "ssrf_detection_middleware"
 
             ::Excon.defaults[:middlewares].insert(0, SSRFDetectionMiddleware)
 

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
-require 'excon'
+require "excon"
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
-require_relative '../../utils/http/url_encoded'
-require_relative '../../utils/http/body'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/url_encoded"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -25,9 +27,9 @@ module Datadog
             headers = normalize_headers(data[:headers])
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => url,
-              'server.io.net.request.method' => data[:method].to_s.upcase,
-              'server.io.net.request.headers' => headers
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => data[:method].to_s.upcase,
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -39,8 +41,8 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data[:body], content_type: headers['content-type'])
-              ephemeral_data['server.io.net.request.body'] = body if body
+              body = parse_request_body(data[:body], content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -57,18 +59,18 @@ module Datadog
             headers = normalize_headers(data.dig(:response, :headers))
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => data.dig(:response, :status).to_s,
-              'server.io.net.response.headers' => headers
+              "server.io.net.response.status" => data.dig(:response, :status).to_s,
+              "server.io.net.response.headers" => headers,
             }
 
-            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?('location')
+            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?("location")
             if is_redirect && data[SAMPLE_BODY_KEY]
-              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers['location']).to_s
+              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers["location"]).to_s
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
-              ephemeral_data['server.io.net.response.body'] = body if body
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -89,15 +91,68 @@ module Datadog
             data[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def request_url(data)
-            klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
+            klass = (data[:scheme] == "https") ? URI::HTTPS : URI::HTTP
             klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
           end
 
@@ -105,7 +160,7 @@ module Datadog
             return {} if headers.nil? || headers.empty?
 
             headers.each_with_object({}) do |(key, value), memo|
-              memo[key.downcase] = value.is_a?(Array) ? value.join(', ') : value
+              memo[key.downcase] = value.is_a?(Array) ? value.join(", ") : value
             end
           end
 

data/lib/datadog/appsec/contrib/faraday/integration.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
+require_relative "../integration"
 
-require_relative 'patcher'
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -12,12 +12,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('0.14.0')
+          MINIMUM_VERSION = Gem::Version.new("0.14.0")
 
           register_as :faraday, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['faraday']&.version
+            Gem.loaded_specs["faraday"]&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/faraday/patcher.rb

@@ -17,9 +17,9 @@ module Datadog
           end
 
           def patch
-            require_relative 'ssrf_detection_middleware'
-            require_relative 'connection_patch'
-            require_relative 'rack_builder_patch'
+            require_relative "ssrf_detection_middleware"
+            require_relative "connection_patch"
+            require_relative "rack_builder_patch"
 
             ::Faraday::Middleware.register_middleware(datadog_appsec: SSRFDetectionMiddleware)
             configure_default_faraday_connection
@@ -28,7 +28,7 @@ module Datadog
           end
 
           def configure_default_faraday_connection
-            if target_version&.>= Gem::Version.new('1.0.0')
+            if target_version&.>= Gem::Version.new("1.0.0")
               # Patch the default connection (e.g. +Faraday.get+)
               ::Faraday.default_connection.use(:datadog_appsec)
 

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
-require_relative '../../utils/http/media_type'
-require_relative '../../utils/http/body'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -23,9 +24,9 @@ module Datadog
             headers = normalize_headers(env.request_headers)
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => url,
-              'server.io.net.request.method' => env.method.to_s.upcase,
-              'server.io.net.request.headers' => headers
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => env.method.to_s.upcase,
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -37,8 +38,8 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
-              ephemeral_data['server.io.net.request.body'] = body if body
+              body = parse_request_body(env.body, content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -54,18 +55,18 @@ module Datadog
             headers = normalize_headers(env.response_headers)
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => env.status.to_s,
-              'server.io.net.response.headers' => headers
+              "server.io.net.response.status" => env.status.to_s,
+              "server.io.net.response.headers" => headers,
             }
 
-            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
+            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?("location")
             if is_redirect && env[SAMPLE_BODY_KEY]
-              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
+              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers["location"]).to_s
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
-              ephemeral_data['server.io.net.response.body'] = body if body
+              body = parse_response_body(env.body, headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -82,11 +83,64 @@ module Datadog
             env[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def normalize_headers(headers)

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require_relative 'gateway/multiplex'
-require_relative '../../instrumentation/gateway'
+require "json"
+require_relative "gateway/multiplex"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -16,7 +16,7 @@ module Datadog
 
             gateway_multiplex = Gateway::Multiplex.new(multiplex)
 
-            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push("graphql.multiplex", gateway_multiplex) do
               super
             end