datadog 2.21.0 → 2.38.0

data/lib/datadog/appsec/context.rb

@@ -1,16 +1,36 @@
 # frozen_string_literal: true
 
+require_relative 'counter_sampler'
 require_relative 'metrics'
+require_relative 'security_event'
 
 module Datadog
   module AppSec
-    # This class accumulates the context over the request life-cycle and exposes
-    # interface sufficient for instrumentation to perform threat detection.
+    # Request-bound context providing threat detection interface.
+    #
+    # Activated at the start of a request (see `Contrib::Rack::RequestMiddleware`)
+    # and shared across all instrumentations within that request's lifecycle.
+    #
+    # Accumulates security events, metrics, and state needed for coordinated
+    # threat detection.
+    #
+    # @api private
     class Context
-      ActiveContextError = Class.new(StandardError)
+      # Steep: https://github.com/soutaro/steep/issues/1880
+      ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
 
       # TODO: add delegators for active trace span
-      attr_reader :trace, :span, :events
+      attr_reader :trace, :span, :metrics
+
+      # Shared mutable storage for counters, flags, and data accumulated during
+      # the request's lifecycle.
+      #
+      # NOTE: This attribute is a subject to change, but in a current form
+      #       it's a `Hash`-like structure.
+      attr_reader :state
+
+      # Sampler for downstream HTTP request/response body analysis.
+      attr_reader :downstream_body_sampler
 
       class << self
         def activate(context)
@@ -34,10 +54,16 @@ module Datadog
       def initialize(trace, span, waf_runner)
         @trace = trace
         @span = span
-        @events = []
         @waf_runner = waf_runner
         @metrics = Metrics::Collector.new
-        @interrupted = false
+        @downstream_body_sampler = CounterSampler.new(
+          Datadog.configuration.appsec.api_security.downstream_body_analysis.sample_rate
+        )
+        @state = {
+          events: [],
+          interrupted: false,
+          downstream_body_analyzed_count: 0
+        }
       end
 
       def run_waf(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
@@ -47,21 +73,25 @@ module Datadog
         result
       end
 
-      def run_rasp(type, persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+      def run_rasp(type, persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT, phase: nil)
         result = @waf_runner.run(persistent_data, ephemeral_data, timeout)
 
-        Metrics::Telemetry.report_rasp(type, result)
-        @metrics.record_rasp(result)
+        Metrics::Telemetry.report_rasp(type, result, phase: phase)
+        @metrics.record_rasp(result, type: type, phase: phase)
 
         result
       end
 
+      def events
+        @state[:events]
+      end
+
       def mark_as_interrupted!
-        @interrupted = true
+        @state[:interrupted] = true
       end
 
       def interrupted?
-        @interrupted
+        @state[:interrupted]
       end
 
       def waf_runner_ruleset_version
@@ -72,8 +102,14 @@ module Datadog
         @waf_runner.waf_addresses
       end
 
-      def extract_schema
-        @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
+      def extract_schema!
+        persistent_data = {'waf.context.processor' => {'extract-schema' => true}}
+        waf_result = run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+        security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
+
+        @state[:schema_extracted] = security_event.schema?
+
+        events.push(security_event)
       end
 
       def export_metrics
@@ -81,12 +117,14 @@ module Datadog
 
         Metrics::Exporter.export_waf_metrics(@metrics.waf, @span)
         Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
+        Metrics::Exporter.export_downstream_response_metrics(@metrics.downstream_responses, @span)
       end
 
       def export_request_telemetry
         return if @trace.nil?
 
         Metrics::TelemetryExporter.export_waf_request_metrics(@metrics.waf, self)
+        Metrics::TelemetryExporter.export_api_security_metrics(self)
       end
 
       def finalize!

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
 
 module Datadog
@@ -31,7 +32,8 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SQLI, {}, ephemeral_data, waf_timeout)
 
             if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag(context, result)
+              TraceKeeper.keep!(context.trace) if result.keep?
 
               context.events.push(
                 AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)

data/lib/datadog/appsec/contrib/active_record/integration.rb

@@ -24,7 +24,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/active_record/patcher.rb

@@ -11,7 +11,7 @@ module Datadog
           module_function
 
           def patched?
-            Patcher.instance_variable_get(:@patched)
+            !!Patcher.instance_variable_get(:@patched)
           end
 
           def target_version
@@ -61,6 +61,7 @@ module Datadog
             end
 
             ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
+            Patcher.instance_variable_set(:@patched, true)
           end
 
           def patch_mysql2_adapter
@@ -73,6 +74,7 @@ module Datadog
             end
 
             ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
+            Patcher.instance_variable_set(:@patched, true)
           end
 
           def patch_postgresql_adapter
@@ -93,6 +95,7 @@ module Datadog
             end
 
             ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/aws_lambda/gateway/watcher.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative '../waf_addresses'
+require_relative '../../../event'
+require_relative '../../../trace_keeper'
+require_relative '../../../security_event'
+require_relative '../../../instrumentation/gateway'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        module Gateway
+          module Watcher
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
+
+                watch_request(gateway)
+                watch_response(gateway)
+              end
+
+              def watch_request(gateway = Instrumentation.gateway)
+                gateway.watch('aws_lambda.request.start') do |stack, payload|
+                  context = payload.context
+                  next stack.call(payload) unless context
+
+                  persistent_data = WAFAddresses.from_request(payload.data)
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match? || !result.attributes.empty?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+                  end
+
+                  if result.match?
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(payload)
+                end
+              end
+
+              def watch_response(gateway = Instrumentation.gateway)
+                gateway.watch('aws_lambda.response.start') do |stack, payload|
+                  context = payload.context
+                  next stack.call(payload) unless context
+
+                  persistent_data = WAFAddresses.from_response(payload.data)
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match?
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(payload)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/integration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'patcher'
+require_relative '../integration'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          register_as :aws_lambda, auto_patch: false
+
+          # NOTE: AWS Lambda is a runtime environment, not an installable gem
+          def self.version
+            nil
+          end
+
+          def self.loaded?
+            true
+          end
+
+          def self.compatible?
+            super
+          end
+
+          def self.auto_instrument?
+            false
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/patcher.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../monitor'
+require_relative 'gateway/watcher'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        module Patcher
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            Monitor::Gateway::Watcher.watch
+            Gateway::Watcher.watch
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+require_relative '../../utils/http/media_type'
+require_relative '../../utils/http/body'
+require_relative '../../../core/utils/base64_codec'
+require_relative '../../../core/header_collection'
+require_relative '../../../tracing/client_ip'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module AwsLambda
+        # Extracts WAF input addresses from normalized AWS Lambda API Gateway event payloads.
+        # @api private
+        module WAFAddresses
+          BASE64_CHARS_PER_GROUP = 4
+          BASE64_BYTES_PER_GROUP = 3
+          BASE64_PADDING_BYTE = "=".ord
+
+          module_function
+
+          def from_request(payload)
+            return {} if payload.nil? || payload.empty?
+
+            headers = parse_headers(payload)
+            data = {
+              'server.request.cookies' => parse_cookies(payload, headers),
+              'server.request.query' => payload['query'],
+              'server.request.uri.raw' => build_fullpath(payload),
+              'server.request.headers' => headers,
+              'server.request.headers.no_cookies' => headers.dup.tap { |h| h.delete('cookie') },
+              'server.request.method' => payload['method'],
+              'server.request.body' => parse_body(payload, headers),
+              'server.request.body.byte_length' => body_byte_length(payload),
+              'server.request.path_params' => payload['path_params'],
+              'http.client_ip' => extract_client_ip(payload['source_ip'], headers)
+            }
+
+            data.compact!
+            data
+          end
+
+          def from_response(payload)
+            return {} if payload.nil? || payload.empty?
+
+            headers = parse_headers(payload)
+            data = {
+              'server.response.status' => payload['status_code']&.to_s,
+              'server.response.headers' => headers,
+              'server.response.headers.no_cookies' => headers.dup.tap { |h| h.delete('set-cookie') },
+              'server.response.body' => parse_body(payload, headers),
+              'server.response.body.byte_length' => body_byte_length(payload)
+            }
+
+            data.compact!
+            data
+          end
+
+          def parse_headers(payload)
+            (payload['headers'] || {}).each_with_object({}) do |(key, value), hash|
+              hash[key.downcase] = value
+            end
+          end
+
+          def parse_cookies(payload, headers)
+            raw_pairs = payload['cookies'] || headers['cookie']&.split(';')
+            return unless raw_pairs
+
+            raw_pairs.each_with_object({}) do |pair, hash|
+              name, value = pair.strip.split('=', 2)
+              hash[name] = value if name
+            end
+          end
+
+          def build_fullpath(payload)
+            path = payload['path']
+            return unless path
+
+            query_string = build_query_string(payload)
+            query_string ? "#{path}?#{query_string}" : path
+          end
+
+          def build_query_string(payload)
+            query_string = payload['query_string']
+            return query_string if query_string && !query_string.empty?
+
+            query = payload['query']
+            return if query.nil? || query.empty?
+
+            URI.encode_www_form(query)
+          end
+
+          def extract_client_ip(remote_ip, headers)
+            header_collection = Datadog::Core::HeaderCollection.from_hash(headers)
+            Datadog::Tracing::ClientIp.extract_client_ip(header_collection, remote_ip)
+          end
+
+          def parse_body(payload, headers)
+            body = payload['body']
+            return unless body
+
+            if (byte_length = body_byte_length(payload))
+              return if byte_length > Datadog.configuration.appsec.body_parsing_size_limit
+            end
+
+            content_type = headers['content-type']
+            return unless content_type
+
+            media_type = AppSec::Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            body = Core::Utils::Base64Codec.strict_decode64(body) if payload['base64_encoded']
+            AppSec::Utils::HTTP::Body.parse(body, media_type: media_type)
+          rescue ArgumentError => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to decode base64 body')
+
+            nil
+          end
+
+          def body_byte_length(payload)
+            body = payload['body']
+
+            return unless body
+            return body.bytesize unless payload['base64_encoded']
+
+            # NOTE: Base64 packs every 3 bytes into 4 characters and pads the last
+            #       group with up to two "=" bytes. The decoded length is therefore
+            #       derivable from the encoded length, letting us measure the raw
+            #       body size without allocating the decoded string.
+            padding = 0
+            if body.getbyte(-1) == BASE64_PADDING_BYTE
+              padding = 1
+              padding = 2 if body.getbyte(-2) == BASE64_PADDING_BYTE
+            end
+
+            body.bytesize / BASE64_CHARS_PER_GROUP * BASE64_BYTES_PER_GROUP - padding
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/integration.rb

@@ -24,7 +24,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb

@@ -18,9 +18,10 @@ module Datadog
               return result unless AppSec.enabled?
               return result if @_datadog_appsec_skip_track_login_event
               return result unless Configuration.auto_user_instrumentation_enabled?
-              return result unless AppSec.active_context
 
               context = AppSec.active_context
+              return result unless context
+
               if context.trace.nil? || context.span.nil?
                 Datadog.logger.debug { 'AppSec: unable to track signin events, due to missing trace or span' }
                 return result

data/lib/datadog/appsec/contrib/excon/integration.rb

@@ -24,7 +24,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/excon/patcher.rb

@@ -9,7 +9,7 @@ module Datadog
           module_function
 
           def patched?
-            Patcher.instance_variable_get(:@patched)
+            !!Patcher.instance_variable_get(:@patched)
           end
 
           def target_version
@@ -20,6 +20,8 @@ module Datadog
             require_relative 'ssrf_detection_middleware'
 
             ::Excon.defaults[:middlewares].insert(0, SSRFDetectionMiddleware)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end