datadog 2.8.0 → 2.39.0

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -1,31 +1,27 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
-require_relative 'gateway/request'
-require_relative 'gateway/response'
-require_relative '../../instrumentation/gateway'
-require_relative '../../processor'
-require_relative '../../response'
+require_relative "response_body"
+require_relative "gateway/request"
+require_relative "gateway/response"
 
-require_relative '../../../tracing/client_ip'
-require_relative '../../../tracing/contrib/rack/header_collection'
+require_relative "../../event"
+require_relative "../../response"
+require_relative "../../api_security"
+require_relative "../../default_header_tags"
+require_relative "../../route_normalizer"
+require_relative "../../security_event"
+require_relative "../../instrumentation/gateway"
+
+require_relative "../../../core/header_collection"
+require_relative "../../../tracing/client_ip"
+require_relative "../../../tracing/contrib/rack/header_collection"
 
 module Datadog
   module AppSec
     module Contrib
       module Rack
-        WAF_VENDOR_HEADERS_TAGS = %w[
-          X-Amzn-Trace-Id
-          Cloudfront-Viewer-Ja3-Fingerprint
-          Cf-Ray
-          X-Cloud-Trace-Context
-          X-Appgw-Trace-id
-          X-SigSci-RequestID
-          X-SigSci-Tags
-          Akamai-User-Risk
-        ].map(&:downcase).freeze
-
         # Topmost Rack middleware for AppSec
         # This should be inserted just below Datadog::Tracing::Contrib::Rack::TraceMiddleware
         class RequestMiddleware
@@ -33,99 +29,99 @@ module Datadog
             @app = app
 
             @oneshot_tags_sent = false
-            @rack_headers = {}
           end
 
-          # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          # rubocop:disable Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
             boot = Datadog::Core::Remote::Tie.boot
             Datadog::Core::Remote::Tie::Tracing.tag(boot, active_span)
 
-            processor = nil
-            ready = false
-            scope = nil
-
             # For a given request, keep using the first Rack stack scope for
             # nested apps. Don't set `context` local variable so that on popping
             # out of this nested stack we don't finalize the parent's context
-            return @app.call(env) if active_scope(env)
-
-            Datadog::AppSec.reconfigure_lock do
-              processor = Datadog::AppSec.processor
+            return @app.call(env) if active_context(env)
 
-              if !processor.nil? && processor.ready?
-                scope = Datadog::AppSec::Scope.activate_scope(active_trace, active_span, processor)
-                env[Datadog::AppSec::Ext::SCOPE_KEY] = scope
-                ready = true
-              end
-            end
+            security_engine = Datadog::AppSec.security_engine
 
             # TODO: handle exceptions, except for @app.call
+            return @app.call(env) unless security_engine
 
-            return @app.call(env) unless ready
+            ctx = Datadog::AppSec::Context.activate(
+              Datadog::AppSec::Context.new(active_trace, active_span, security_engine.new_runner)
+            )
+            env[Datadog::AppSec::Ext::CONTEXT_KEY] = ctx
 
-            gateway_request = Gateway::Request.new(env)
+            add_appsec_tags(ctx)
+            add_request_tags(ctx, env)
 
-            add_appsec_tags(processor, scope)
-            add_request_tags(scope, env)
+            http_response = nil
+            gateway_request = Gateway::Request.new(env)
+            gateway_response = nil
 
-            request_return, request_response = catch(::Datadog::AppSec::Ext::INTERRUPT) do
-              Instrumentation.gateway.push('rack.request', gateway_request) do
+            interrupt_params = catch(::Datadog::AppSec::Ext::INTERRUPT) do
+              # TODO: This event should be renamed into `rack.request.start` to
+              #       reflect that it's the beginning of the request-cycle
+              http_response, _gateway_request = Instrumentation.gateway.push("rack.request", gateway_request) do
                 @app.call(env)
               end
-            end
 
-            if request_response
-              blocked_event = request_response.find { |action, _options| action == :block }
-              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
-            end
+              gateway_response = Gateway::Response.new(
+                http_response[2], http_response[0], http_response[1], context: ctx
+              )
 
-            gateway_response = Gateway::Response.new(
-              request_return[2],
-              request_return[0],
-              request_return[1],
-              scope: scope,
-            )
+              Instrumentation.gateway.push("rack.request.finish", gateway_request)
+              Instrumentation.gateway.push("rack.response", gateway_response)
 
-            _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
+              nil
+            end
 
-            result = scope.processor_context.extract_schema
+            add_normalized_route_tag(ctx, env)
 
-            if result
-              scope.processor_context.events << {
-                trace: scope.trace,
-                span: scope.service_entry_span,
-                waf_result: result,
-              }
+            if interrupt_params
+              ctx.mark_as_interrupted!
+              http_response = AppSec::Response.from_interrupt_params(interrupt_params, env["HTTP_ACCEPT"]).to_rack
             end
 
-            scope.processor_context.events.each do |e|
-              e[:response] ||= gateway_response
-              e[:request]  ||= gateway_request
+            # NOTE: This is not optimal, but in the current implementation
+            #       `gateway_response` is a container to dispatch response event
+            #       and in case of interruption it suppose to be `nil`.
+            #
+            #       `http_response` is a real response object in both cases, but
+            #       to save us some computations, we will use already pre-computed
+            #       `gateway_response` instead of re-creating it.
+            #
+            # WARNING: This part will be refactored.
+            tmp_response = if interrupt_params
+              Gateway::Response.new(http_response[2], http_response[0], http_response[1], context: ctx)
+            else
+              gateway_response
             end
 
-            AppSec::Event.record(scope.service_entry_span, *scope.processor_context.events)
-
-            if response_response
-              blocked_event = response_response.find { |action, _options| action == :block }
-              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
+            if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
+                AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
+              ctx.extract_schema!
             end
 
-            request_return
+            AppSec::Event.record(ctx, request: gateway_request)
+
+            add_response_tags(ctx, tmp_response)
+            http_response
           ensure
-            if scope
-              add_waf_runtime_tags(scope)
-              Datadog::AppSec::Scope.deactivate_scope
+            if ctx
+              ctx.export_metrics
+              ctx.export_request_telemetry
+
+              Datadog::AppSec::Context.deactivate
             end
           end
-          # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          # rubocop:enable Metrics/MethodLength
 
           private
 
-          def active_scope(env)
-            env[Datadog::AppSec::Ext::SCOPE_KEY]
+          def active_context(env)
+            env[Datadog::AppSec::Ext::CONTEXT_KEY]
           end
 
           def active_trace
@@ -144,32 +140,26 @@ module Datadog
             Datadog::Tracing.active_span
           end
 
-          def add_appsec_tags(processor, scope)
-            span = scope.service_entry_span
-            trace = scope.trace
+          # standard:disable Metrics/MethodLength
+          def add_appsec_tags(context)
+            span = context.span
+            trace = context.trace
 
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
-            span.set_tag('_dd.runtime_family', 'ruby')
-            span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+            span.set_tag("_dd.runtime_family", "ruby")
+            span.set_tag("_dd.appsec.waf.version", Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            if processor.diagnostics
-              diagnostics = processor.diagnostics
+            if context.waf_runner_ruleset_version
+              span.set_tag("_dd.appsec.event_rules.version", context.waf_runner_ruleset_version)
 
-              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
-
-              unless @oneshot_tags_sent
+              unless oneshot_tags_sent?
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
-                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
+                span.set_tag("_dd.appsec.event_rules.addresses", JSON.dump(context.waf_runner_known_addresses))
 
                 # Ensure these tags reach the backend
                 trace.keep!
@@ -180,45 +170,57 @@ module Datadog
               end
             end
           end
+          # standard:enable Metrics/MethodLength
 
-          def add_request_tags(scope, env)
-            span = scope.service_entry_span
-
+          def add_request_tags(context, env)
+            span = context.span
             return unless span
 
-            # Always add WAF vendors headers
-            WAF_VENDOR_HEADERS_TAGS.each do |lowercase_header|
-              rack_header = to_rack_header(lowercase_header)
-              span.set_tag("http.request.headers.#{lowercase_header}", env[rack_header]) if env[rack_header]
-            end
-
-            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
-              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+            headers = Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+            AppSec::DefaultHeaderTags.tag_request(span, headers)
 
+            if span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
               # always collect client ip, as this is part of AppSec provided functionality
               Datadog::Tracing::ClientIp.set_client_ip_tag!(
-                span,
-                headers: request_header_collection,
-                remote_ip: env['REMOTE_ADDR']
+                span, headers: headers, remote_ip: env["REMOTE_ADDR"]
               )
             end
           end
 
-          def add_waf_runtime_tags(scope)
-            span = scope.service_entry_span
-            context = scope.processor_context
+          def add_response_tags(context, response)
+            span = context.span
+            return unless span
+
+            AppSec::DefaultHeaderTags.tag_response(
+              span, Datadog::Core::HeaderCollection.from_hash(response.headers)
+            )
+
+            unless response.headers.key?("content-length")
+              length = ResponseBody.content_length(response.body)
+              span.set_tag("http.response.headers.content-length", length.to_s) if length
+            end
+          end
+
+          def add_normalized_route_tag(context, env)
+            return unless AppSec::APISecurity.enabled?
+
+            span = context.span
+            return unless span
 
-            return unless span && context
+            pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
+            return unless pattern
 
-            span.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+            # NOTE: To build full path that covers mounted engines we need to add
+            #       pre-computed by Tracer route path tag to the normalized route
+            prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env["SCRIPT_NAME"]
+            normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
+            return unless normalized_route
 
-            # these tags expect time in us
-            span.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
-            span.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+            span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
           end
 
-          def to_rack_header(header)
-            @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)
+          def oneshot_tags_sent?
+            @oneshot_tags_sent
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/response_body.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module ResponseBody
+          # NOTE: We compute content length only for fixed-size response bodies,
+          #       ignoring streaming bodies to avoid buffering.
+          #
+          #       On Rack 3.x, `body.to_ary` on a BodyProxy triggers `close` on all
+          #       nested BodyProxy layers. This is safe because web servers, like Puma
+          #       handles already-closed bodies (its own `to_ary` becomes a no-op).
+          #
+          # @see Puma::Response#prepare_response
+          # @see https://github.com/puma/puma/blob/b1271222cbf21868f3fb64154caa4d03936a7b9e/lib/puma/response.rb#L165-L168
+          def self.content_length(body)
+            return unless body.respond_to?(:to_ary)
+
+            # NOTE: When `to_ary` exists but returns `nil`, the body will be
+            #       transferred in chunks and we can't compute content length
+            #       without buffering it.
+            body_ary = body.to_ary
+            return unless body_ary.is_a?(Array)
+
+            body_ary.sum { |part| part.is_a?(String) ? part.bytesize : 0 }
+          rescue => e
+            AppSec.telemetry.report(e, description: "AppSec: Failed to compute body content length")
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../rack/input_peeker"
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec
@@ -43,22 +44,60 @@ module Datadog
               # usually Hash<String,String> but can be a more complex
               # Hash<String,String||Array||Hash> when e.g coming from JSON or
               # with Rails advanced param square bracket parsing
-              body = request.env['action_dispatch.request.request_parameters']
+              body = request.env["action_dispatch.request.request_parameters"]
 
               return if body.nil?
+              return body unless request.env["action_dispatch.request.path_parameters"]
 
               body.reject do |k, _v|
-                request.env['action_dispatch.request.path_parameters'].key?(k)
+                request.env["action_dispatch.request.path_parameters"].key?(k)
               end
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request body: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: "AppSec: Failed to parse request body")
+
+              nil
             end
 
             def route_params
               excluded = [:controller, :action]
 
-              request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+              request.env.fetch("action_dispatch.request.path_parameters", {}).reject do |k, _v|
                 excluded.include?(k)
               end
             end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       raw posted data, raw form vars, size if known, raw
+            #       Content-Length, then buffering to the limit if unknown-length
+            def body_bytesize(limit)
+              raw_body = env["RAW_POST_DATA"]
+              return raw_body.bytesize if raw_body
+
+              form_vars = env["rack.request.form_vars"]
+              return form_vars.bytesize if form_vars
+
+              io = request.body
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: Read raw `CONTENT_LENGTH` as {ActionDispatch::Request#content_length}
+              #       drains `rack.input` into `RAW_POST_DATA` on chunked Transfer-Encoding
+              content_length = env["CONTENT_LENGTH"]
+              return content_length.to_i if content_length
+
+              # NOTE: An already-read body (e.g. late-parsed multipart on Rack 3+) peeks
+              #       as 0, so we skip byte_length but still collect the parsed body
+              begin
+                Rack::InputPeeker.peek_bytesize(env, limit: limit)
+              rescue => e
+                Datadog.logger.debug { "AppSec: Failed to measure Rails request body: #{e.class}: #{e.message}" }
+                nil
+              end
+            end
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway'
-require_relative '../../../reactive/operation'
-require_relative '../reactive/action'
-require_relative '../../../event'
+require_relative "../../../event"
+require_relative "../../../trace_keeper"
+require_relative "../../../security_event"
+require_relative "../../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -17,47 +17,72 @@ module Datadog
                 gateway = Instrumentation.gateway
 
                 watch_request_action(gateway)
+                watch_response_body_json(gateway)
               end
 
               def watch_request_action(gateway = Instrumentation.gateway)
-                gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
-                  block = false
-
-                  event = nil
-                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
-
-                  AppSec::Reactive::Operation.new('rails.request.action') do |op|
-                    Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result|
-                      if result.status == :match
-                        # TODO: should this hash be an Event instance instead?
-                        event = {
-                          waf_result: result,
-                          trace: scope.trace,
-                          span: scope.service_entry_span,
-                          request: gateway_request,
-                          actions: result.actions
-                        }
-
-                        # We want to keep the trace in case of security event
-                        scope.trace.keep! if scope.trace
-                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                        scope.processor_context.events << event
+                gateway.watch("rails.request.action") do |stack, gateway_request|
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+
+                  persistent_data = {
+                    "server.request.path_params" => gateway_request.route_params
+                  }
+
+                  unless limit.zero?
+                    byte_length = gateway_request.body_bytesize(limit)
+
+                    if byte_length
+                      # NOTE: Params may be parsed before this hook, leaving the
+                      #       body stream at EOF. Keep byte_length unset for that
+                      #       case, but still inspect cached params
+                      persistent_data["server.request.body.byte_length"] = byte_length if byte_length.positive?
+
+                      if byte_length <= limit
+                        body = gateway_request.parsed_body
+                        persistent_data["server.request.body"] = body unless body.nil? || body.empty?
                       end
                     end
+                  end
+
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
 
-                    block = Rails::Reactive::Action.publish(op, gateway_request)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
-                  next [nil, [[:block, event]]] if block
+                  stack.call(gateway_request.request)
+                end
+              end
+
+              def watch_response_body_json(gateway = Instrumentation.gateway)
+                gateway.watch("rails.response.body.json") do |stack, container|
+                  context = container.context
+
+                  persistent_data = {
+                    "server.response.body" => container.data
+                  }
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
 
-                  ret, res = stack.call(gateway_request.request)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
 
-                  if event
-                    res ||= []
-                    res << [:monitor, event]
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
-                  [ret, res]
+                  stack.call(container)
                 end
               end
             end

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
+require_relative "../integration"
 
-require_relative 'patcher'
-require_relative 'request_middleware'
+require_relative "patcher"
+require_relative "request_middleware"
 
 module Datadog
   module AppSec
@@ -13,12 +13,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('4')
+          MINIMUM_VERSION = Gem::Version.new("4")
 
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
+            Gem.loaded_specs["railties"]&.version
           end
 
           def self.loaded?
@@ -26,7 +26,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?