datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/contrib/aws_lambda/gateway/watcher.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative '../waf_addresses'
+require_relative '../../../event'
+require_relative '../../../trace_keeper'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        module Gateway
+          module Watcher
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
+
+                watch_request(gateway)
+                watch_response(gateway)
+              end
+
+              def watch_request(gateway = Instrumentation.gateway)
+                gateway.watch('aws_lambda.request.start') do |stack, payload|
+                  context = payload.context
+                  next stack.call(payload) unless context
+
+                  persistent_data = WAFAddresses.from_request(payload.data)
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match? || !result.attributes.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
+
+                  if result.match?
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(payload)
+                end
+              end
+
+              def watch_response(gateway = Instrumentation.gateway)
+                gateway.watch('aws_lambda.response.start') do |stack, payload|
+                  context = payload.context
+                  next stack.call(payload) unless context
+
+                  persistent_data = WAFAddresses.from_response(payload.data)
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match?
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(payload)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/integration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'patcher'
+require_relative '../integration'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          register_as :aws_lambda, auto_patch: false
+
+          # NOTE: AWS Lambda is a runtime environment, not an installable gem
+          def self.version
+            nil
+          end
+
+          def self.loaded?
+            true
+          end
+
+          def self.compatible?
+            super
+          end
+
+          def self.auto_instrument?
+            false
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/patcher.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../monitor'
+require_relative 'gateway/watcher'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        module Patcher
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            Monitor::Gateway::Watcher.watch
+            Gateway::Watcher.watch
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+require_relative '../../utils/http/media_type'
+require_relative '../../utils/http/body'
+require_relative '../../../core/utils/base64_codec'
+require_relative '../../../core/header_collection'
+require_relative '../../../tracing/client_ip'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        # Extracts WAF input addresses from normalized AWS Lambda API Gateway event payloads.
+        # @api private
+        module WAFAddresses
+          BASE64_CHARS_PER_GROUP = 4
+          BASE64_BYTES_PER_GROUP = 3
+          BASE64_PADDING_BYTE = "=".ord
+
+          module_function
+
+          def from_request(payload)
+            return {} if payload.nil? || payload.empty?
+
+            headers = parse_headers(payload)
+            data = {
+              'server.request.cookies' => parse_cookies(payload, headers),
+              'server.request.query' => payload['query'],
+              'server.request.uri.raw' => build_fullpath(payload),
+              'server.request.headers' => headers,
+              'server.request.headers.no_cookies' => headers.dup.tap { |h| h.delete('cookie') },
+              'server.request.method' => payload['method'],
+              'server.request.body' => parse_body(payload, headers),
+              'server.request.body.byte_length' => body_byte_length(payload),
+              'server.request.path_params' => payload['path_params'],
+              'http.client_ip' => extract_client_ip(payload['source_ip'], headers)
+            }
+
+            data.compact!
+            data
+          end
+
+          def from_response(payload)
+            return {} if payload.nil? || payload.empty?
+
+            headers = parse_headers(payload)
+            data = {
+              'server.response.status' => payload['status_code']&.to_s,
+              'server.response.headers' => headers,
+              'server.response.headers.no_cookies' => headers.dup.tap { |h| h.delete('set-cookie') },
+              'server.response.body' => parse_body(payload, headers),
+              'server.response.body.byte_length' => body_byte_length(payload)
+            }
+
+            data.compact!
+            data
+          end
+
+          def parse_headers(payload)
+            (payload['headers'] || {}).each_with_object({}) do |(key, value), hash|
+              hash[key.downcase] = value
+            end
+          end
+
+          def parse_cookies(payload, headers)
+            raw_pairs = payload['cookies'] || headers['cookie']&.split(';')
+            return unless raw_pairs
+
+            raw_pairs.each_with_object({}) do |pair, hash|
+              name, value = pair.strip.split('=', 2)
+              hash[name] = value if name
+            end
+          end
+
+          def build_fullpath(payload)
+            path = payload['path']
+            return unless path
+
+            query_string = build_query_string(payload)
+            query_string ? "#{path}?#{query_string}" : path
+          end
+
+          def build_query_string(payload)
+            query_string = payload['query_string']
+            return query_string if query_string && !query_string.empty?
+
+            query = payload['query']
+            return if query.nil? || query.empty?
+
+            URI.encode_www_form(query)
+          end
+
+          def extract_client_ip(remote_ip, headers)
+            header_collection = Datadog::Core::HeaderCollection.from_hash(headers)
+            Datadog::Tracing::ClientIp.extract_client_ip(header_collection, remote_ip)
+          end
+
+          def parse_body(payload, headers)
+            body = payload['body']
+            return unless body
+
+            if (byte_length = body_byte_length(payload))
+              return if byte_length > Datadog.configuration.appsec.body_parsing_size_limit
+            end
+
+            content_type = headers['content-type']
+            return unless content_type
+
+            media_type = AppSec::Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            body = Core::Utils::Base64Codec.strict_decode64(body) if payload['base64_encoded']
+            AppSec::Utils::HTTP::Body.parse(body, media_type: media_type)
+          rescue ArgumentError => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to decode base64 body')
+
+            nil
+          end
+
+          def body_byte_length(payload)
+            body = payload['body']
+
+            return unless body
+            return body.bytesize unless payload['base64_encoded']
+
+            # NOTE: Base64 packs every 3 bytes into 4 characters and pads the last
+            #       group with up to two "=" bytes. The decoded length is therefore
+            #       derivable from the encoded length, letting us measure the raw
+            #       body size without allocating the decoded string.
+            padding = 0
+            if body.getbyte(-1) == BASE64_PADDING_BYTE
+              padding = 1
+              padding = 2 if body.getbyte(-2) == BASE64_PADDING_BYTE
+            end
+
+            body.bytesize / BASE64_CHARS_PER_GROUP * BASE64_BYTES_PER_GROUP - padding
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/integration.rb

@@ -24,7 +24,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb

@@ -3,6 +3,7 @@
 require_relative '../ext'
 require_relative '../configuration'
 require_relative '../data_extractor'
+require_relative '../../../trace_keeper'
 
 module Datadog
   module AppSec
@@ -17,15 +18,16 @@ module Datadog
               return result unless AppSec.enabled?
               return result if @_datadog_appsec_skip_track_login_event
               return result unless Configuration.auto_user_instrumentation_enabled?
-              return result unless AppSec.active_context
 
               context = AppSec.active_context
+              return result unless context
+
               if context.trace.nil? || context.span.nil?
                 Datadog.logger.debug { 'AppSec: unable to track signin events, due to missing trace or span' }
                 return result
               end
 
-              context.trace.keep!
+              TraceKeeper.keep!(context.trace)
 
               if result
                 record_successful_signin(context, resource)

data/lib/datadog/appsec/contrib/devise/patches/signup_tracking_patch.rb

@@ -3,6 +3,7 @@
 require_relative '../ext'
 require_relative '../configuration'
 require_relative '../data_extractor'
+require_relative '../../../trace_keeper'
 
 module Datadog
   module AppSec
@@ -26,7 +27,7 @@ module Datadog
 
                 next yield(resource) if resource.new_record? && block_given?
 
-                context.trace.keep!
+                TraceKeeper.keep!(context.trace)
                 record_successful_signup(context, resource)
                 Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_SIGNUP)
 

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -92,7 +92,7 @@ module Datadog
             return if value.nil?
             return value.to_s unless anonymize?
 
-            Anonymizer.anonimyze(value.to_s)
+            Anonymizer.anonymize(value.to_s)
           end
 
           def anonymize?

data/lib/datadog/appsec/contrib/excon/integration.rb

@@ -24,7 +24,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/excon/patcher.rb

@@ -9,7 +9,7 @@ module Datadog
           module_function
 
           def patched?
-            Patcher.instance_variable_get(:@patched)
+            !!Patcher.instance_variable_get(:@patched)
           end
 
           def target_version
@@ -20,6 +20,8 @@ module Datadog
             require_relative 'ssrf_detection_middleware'
 
             ::Excon.defaults[:middlewares].insert(0, SSRFDetectionMiddleware)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -1,10 +1,14 @@
-# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 require 'excon'
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/url_encoded'
+require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -12,31 +16,166 @@ module Datadog
       module Excon
         # AppSec Middleware for Excon
         class SSRFDetectionMiddleware < ::Excon::Middleware::Base
-          def request_call(data)
-            return super unless AppSec.rasp_enabled? && AppSec.active_context
+          REDIRECT_STATUS_CODES = (300..399).freeze
+          SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
 
+          def request_call(data)
             context = AppSec.active_context
+            return super unless context && AppSec.rasp_enabled?
 
-            request_url = URI.join("#{data[:scheme]}://#{data[:host]}", data[:path]).to_s
-            ephemeral_data = {'server.io.net.url' => request_url}
+            url = request_url(data)
+            headers = normalize_headers(data[:headers])
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              'server.io.net.url' => url,
+              'server.io.net.request.method' => data[:method].to_s.upcase,
+              'server.io.net.request.headers' => headers
+            }
 
-            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              data[SAMPLE_BODY_KEY] = true
+            else
+              mark_body_sampling!(data, context: context)
+            end
 
-            if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+            if !is_redirect && data[SAMPLE_BODY_KEY]
+              body = parse_request_body(data[:body], content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
 
-              context.events.push(
-                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
-              )
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
+            handle(result, context: context) if result.match?
+
+            super
+          end
 
-              AppSec::ActionsHandler.handle(result.actions)
+          def response_call(data)
+            context = AppSec.active_context
+            return super unless context && AppSec.rasp_enabled?
+
+            headers = normalize_headers(data.dig(:response, :headers))
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              'server.io.net.response.status' => data.dig(:response, :status).to_s,
+              'server.io.net.response.headers' => headers
+            }
+
+            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?('location')
+            if is_redirect && data[SAMPLE_BODY_KEY]
+              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers['location']).to_s
             end
 
+            if !is_redirect && data[SAMPLE_BODY_KEY]
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
+            handle(result, context: context) if result.match?
+
             super
           end
+
+          private
+
+          def mark_body_sampling!(data, context:)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return if context.state[:downstream_body_analyzed_count] >= max
+            return unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            data[SAMPLE_BODY_KEY] = true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
+          def request_url(data)
+            klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
+            klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
+          end
+
+          def normalize_headers(headers)
+            return {} if headers.nil? || headers.empty?
+
+            headers.each_with_object({}) do |(key, value), memo|
+              memo[key.downcase] = value.is_a?(Array) ? value.join(', ') : value
+            end
+          end
+
+          def handle(result, context:)
+            AppSec::Event.tag(context, result)
+            TraceKeeper.keep!(context.trace) if result.keep?
+
+            context.events.push(
+              AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+            )
+
+            AppSec::ActionsHandler.handle(result.actions)
+          end
         end
       end
     end
   end
 end
-# rubocop:enable Naming/FileName

data/lib/datadog/appsec/contrib/faraday/integration.rb

@@ -25,7 +25,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/faraday/patcher.rb

@@ -28,7 +28,7 @@ module Datadog
           end
 
           def configure_default_faraday_connection
-            if target_version >= Gem::Version.new('1.0.0')
+            if target_version&.>= Gem::Version.new('1.0.0')
               # Patch the default connection (e.g. +Faraday.get+)
               ::Faraday.default_connection.use(:datadog_appsec)