datadog 2.28.0 → 2.41.0

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
-require_relative 'patcher'
+require_relative "../integration"
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -11,19 +11,20 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('2.0.19')
+          MINIMUM_VERSION = Gem::Version.new("2.0.19")
 
           AST_NODE_CLASS_NAMES = {
-            field: 'GraphQL::Language::Nodes::Field',
-            directive: 'GraphQL::Language::Nodes::Directive',
-            variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
-            input_object: 'GraphQL::Language::Nodes::InputObject',
+            field: "GraphQL::Language::Nodes::Field",
+            directive: "GraphQL::Language::Nodes::Directive",
+            fragment_spread: "GraphQL::Language::Nodes::FragmentSpread",
+            variable_identifier: "GraphQL::Language::Nodes::VariableIdentifier",
+            input_object: "GraphQL::Language::Nodes::InputObject",
           }.freeze
 
           register_as :graphql, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['graphql']&.version
+            Gem.loaded_specs["graphql"]&.version
           end
 
           def self.loaded?
@@ -31,7 +32,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION && ast_node_classes_defined?
+            !!(super && version&.>=(MINIMUM_VERSION) && ast_node_classes_defined?)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/graphql/patcher.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'gateway/watcher'
+require_relative "gateway/watcher"
 
-if Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version >= Gem::Version.new('2.0.19')
-  require_relative 'appsec_trace'
+if Gem.loaded_specs["graphql"] && Gem.loaded_specs["graphql"].version >= Gem::Version.new("2.0.19")
+  require_relative "appsec_trace"
 end
 
 module Datadog

data/lib/datadog/appsec/contrib/rack/buffered_input.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Wraps a `rack.input` stream with a buffer placed in front of it.
+        # Every read drains the buffer first, then continues from the stream
+        #
+        # NOTE: Forward-only: no rewind, no seek
+        #
+        # NOTE: Rack 3 dropped the rewind requirement from the input stream contract
+        # @see https://github.com/rack/rack/blob/v3.2.6/SPEC.rdoc
+        class BufferedInput
+          # NOTE: Rack's multipart parser reads in 1 MiB chunks, used to bound
+          #       {#each} the same way
+          # @see https://github.com/rack/rack/blob/v3.2.6/lib/rack/multipart/parser.rb#L54
+          READ_BUFSIZE_BYTES = 1_048_576
+
+          def initialize(stream, buffer:)
+            @stream = stream
+            @buffer = buffer
+          end
+
+          def read(length = nil, outbuf = nil)
+            if length.nil?
+              data = @buffer.read(nil, outbuf) || +""
+              more = @stream.read
+
+              data << more if more
+
+              return data
+            end
+
+            data = @buffer.read(length, outbuf)
+
+            if data.nil?
+              more = @stream.read(length, outbuf)
+              return more if more && !more.empty?
+
+              # NOTE: Match `IO#read(length, outbuf)` at EOF. Return nil and clear
+              #       the caller's buffer so stale bytes are not mistaken for data
+              outbuf&.clear
+              return
+            end
+
+            remaining = length - data.bytesize
+            return data if remaining <= 0
+
+            more = @stream.read(remaining)
+            data << more if more
+
+            data
+          end
+
+          def gets
+            line = @buffer.gets
+
+            return @stream.gets if line.nil?
+            return line if line.end_with?("\n")
+
+            more = @stream.gets
+            more ? (line << more) : line
+          end
+
+          def each
+            while (chunk = read(READ_BUFSIZE_BYTES))
+              yield chunk
+            end
+
+            self
+          end
+
+          def close
+            @buffer.close
+          ensure
+            @stream.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -7,38 +7,38 @@ module Datadog
         # Rack integration constants
         module Ext
           COLLECTABLE_REQUEST_HEADERS = [
-            'accept',
-            'akamai-user-risk',
-            'cf-ray',
-            'cloudfront-viewer-ja3-fingerprint',
-            'content-type',
-            'user-agent',
-            'x-amzn-trace-Id',
-            'x-appgw-trace-id',
-            'x-cloud-trace-context',
-            'x-sigsci-requestid',
-            'x-sigsci-tags'
+            "accept",
+            "akamai-user-risk",
+            "cf-ray",
+            "cloudfront-viewer-ja3-fingerprint",
+            "content-type",
+            "user-agent",
+            "x-amzn-trace-id",
+            "x-appgw-trace-id",
+            "x-cloud-trace-context",
+            "x-sigsci-requestid",
+            "x-sigsci-tags",
           ].freeze
 
           IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
-            'accept-encoding',
-            'accept-language',
-            'cf-connecting-ip',
-            'cf-connecting-ipv6',
-            'content-encoding',
-            'content-language',
-            'content-length',
-            'fastly-client-ip',
-            'forwarded',
-            'forwarded-for',
-            'host',
-            'true-client-ip',
-            'via',
-            'x-client-ip',
-            'x-cluster-client-ip',
-            'x-forwarded',
-            'x-forwarded-for',
-            'x-real-ip'
+            "accept-encoding",
+            "accept-language",
+            "cf-connecting-ip",
+            "cf-connecting-ipv6",
+            "content-encoding",
+            "content-language",
+            "content-length",
+            "fastly-client-ip",
+            "forwarded",
+            "forwarded-for",
+            "host",
+            "true-client-ip",
+            "via",
+            "x-client-ip",
+            "x-cluster-client-ip",
+            "x-forwarded",
+            "x-forwarded-for",
+            "x-real-ip",
           ].freeze
         end
       end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
-require_relative '../../../../core/header_collection'
-require_relative '../../../../tracing/client_ip'
+require_relative "../input_peeker"
+require_relative "../../../instrumentation/gateway/argument"
+require_relative "../../../utils/http/url_encoded"
+require_relative "../../../../core/header_collection"
+require_relative "../../../../tracing/client_ip"
 
 module Datadog
   module AppSec
@@ -23,16 +25,7 @@ module Datadog
             end
 
             def query
-              # Downstream libddwaf expects keys and values to be extractable
-              # separately so we can't use [[k, v], ...]. We also want to allow
-              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
-              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
-              request.query_string.split('&').each.with_object({}) do |e, hash|
-                k, v = e.split('=').map { |s| CGI.unescape(s) }
-                hash[k] ||= []
-
-                hash[k] << v
-              end
+              Utils::HTTP::URLEncoded.parse(request.query_string)
             end
 
             def method
@@ -41,12 +34,12 @@ module Datadog
 
             def headers
               result = request.env.each_with_object({}) do |(k, v), h|
-                h[k.delete_prefix('HTTP_').tap(&:downcase!).tap { |s| s.tr!('_', '-') }] = v if k.start_with?('HTTP_')
+                h[k.delete_prefix("HTTP_").tap(&:downcase!).tap { |s| s.tr!("_", "-") }] = v if k.start_with?("HTTP_")
               end
 
-              result['content-type'] = request.content_type if request.content_type
+              result["content-type"] = request.content_type if request.content_type
               # Since Rack 3.1, content-length is nil if the body is empty, but we still want to send it to the WAF.
-              result['content-length'] = request.content_length || '0'
+              result["content-length"] = request.content_length || "0"
               result
             end
 
@@ -75,16 +68,58 @@ module Datadog
             end
 
             def remote_addr
-              env['REMOTE_ADDR']
+              env["REMOTE_ADDR"]
             end
 
             def form_hash
-              # force form data processing
+              # NOTE: Rack populates `rack.request.form_hash` only as a side effect
+              #       of {::Rack::Request#POST}, which reads and parses the body.
               request.POST if request.form_data?
 
-              # usually Hash<String,String> but can be a more complex
-              # Hash<String,String||Array||Hash> when e.g coming from JSON
-              env['rack.request.form_hash']
+              # usually Hash[String, String] but can be a more complex
+              # Hash[String, (String|Array|Hash)] when e.g coming from JSON
+              env["rack.request.form_hash"]
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request body: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: "AppSec: Failed to parse request body")
+
+              nil
+            end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       size if it's known, content-length if provided, and buffering
+            #       to the limit if unknown-length
+            #
+            # WARNING: The buffering path adds overhead for streaming web-servers
+            #          (Rack 3+) when the body length is unknown
+            def body_bytesize(limit)
+              io = request.body
+
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: {::Rack::Request#content_length} is a plain `CONTENT_LENGTH`
+              #       getter forces no body read
+              content_length = request.content_length
+              return content_length.to_i if content_length
+
+              # NOTE: A parsed-body cache without a raw byte count means the input
+              #       stream was consumed before measurement, consider it unknown-length
+              return if env.key?("rack.request.form_hash")
+
+              InputPeeker.peek_bytesize(env, limit: limit)
+            end
+
+            # Whether a request body can be collected without forcing a parse:
+            # either form data parseable on demand, or a body already parsed upstream
+            #
+            # NOTE: Rack does not parse JSON itself, a body parser middleware such as
+            #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
+            def collectable_body?
+              request.form_data? || request.parseable_data? || env.key?("rack.request.form_hash")
             end
 
             def client_ip

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../ext'
-require_relative '../../../event'
-require_relative '../../../trace_keeper'
-require_relative '../../../security_event'
-require_relative '../../../instrumentation/gateway'
+require_relative "../ext"
+require_relative "../../../event"
+require_relative "../../../trace_keeper"
+require_relative "../../../security_event"
+require_relative "../../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -24,17 +24,17 @@ module Datadog
               end
 
               def watch_request(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request', :appsec) do |stack, gateway_request|
+                gateway.watch("rack.request") do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
-                    'server.request.cookies' => gateway_request.cookies,
-                    'server.request.query' => gateway_request.query,
-                    'server.request.uri.raw' => gateway_request.fullpath,
-                    'server.request.headers' => gateway_request.headers,
-                    'server.request.headers.no_cookies' => gateway_request.headers.dup.tap { |h| h.delete('cookie') },
-                    'http.client_ip' => gateway_request.client_ip,
-                    'server.request.method' => gateway_request.method
+                    "server.request.cookies" => gateway_request.cookies,
+                    "server.request.query" => gateway_request.query,
+                    "server.request.uri.raw" => gateway_request.fullpath,
+                    "server.request.headers" => gateway_request.headers,
+                    "server.request.headers.no_cookies" => gateway_request.headers.dup.tap { |h| h.delete("cookie") },
+                    "http.client_ip" => gateway_request.client_ip,
+                    "server.request.method" => gateway_request.method,
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
@@ -57,13 +57,13 @@ module Datadog
               end
 
               def watch_response(gateway = Instrumentation.gateway)
-                gateway.watch('rack.response', :appsec) do |stack, gateway_response|
+                gateway.watch("rack.response") do |stack, gateway_response|
                   context = gateway_response.context
 
                   persistent_data = {
-                    'server.response.status' => gateway_response.status.to_s,
-                    'server.response.headers' => gateway_response.headers,
-                    'server.response.headers.no_cookies' => gateway_response.headers.dup.tap { |h| h.delete('set-cookie') }
+                    "server.response.status" => gateway_response.status.to_s,
+                    "server.response.headers" => gateway_response.headers,
+                    "server.response.headers.no_cookies" => gateway_response.headers.dup.tap { |h| h.delete("set-cookie") },
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
@@ -84,12 +84,33 @@ module Datadog
               end
 
               def watch_request_body(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
+                gateway.watch("rack.request.body") do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data["server.request.body.byte_length"] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data["server.request.body"] = body if body
+                    end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?("rack.request.form_hash")
+                    body = gateway_request.env["rack.request.form_hash"]
+                    persistent_data["server.request.body"] = body if body
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
@@ -113,20 +134,15 @@ module Datadog
               #       somewhere closer to identity related monitor.
               # WARNING: The Gateway is a subject of refactoring
               def watch_request_finish(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
+                gateway.watch("rack.request.finish") do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
-                    next stack.call(gateway_request.request)
-                  end
+                  next stack.call(gateway_request.request) if context.span.nil?
 
                   gateway_request.headers.each do |name, value|
-                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
-                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
-                      next
+                    if context.state[:has_identity_event] && Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      context.span["http.request.headers.#{name}"] ||= value
                     end
-
-                    context.span["http.request.headers.#{name}"] ||= value
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rack/input_peeker.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "stringio"
+require_relative "buffered_input"
+require_relative "../../utils/http/body_reader"
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Peeks at `rack.input` without changing what downstream Rack code can read
+        #
+        # @api private
+        module InputPeeker
+          module_function
+
+          # Peeks at `env['rack.input']` and returns the body bytesize when it
+          # can be measured within the given limit
+          #
+          # NOTE: Over-limit bodies cannot be fully measured, so returning `nil`
+          #       is the tradeoff we accept
+          #
+          # WARNING: For forward-only input, replaces `env['rack.input']` with a
+          #          replay stream over the peeked bytes, preserving the rest
+          #          of the original stream
+          #
+          # @param env [Hash] Rack environment
+          # @param limit [Integer] Maximum body bytesize to measure
+          # @return [Integer, nil] `Integer` bytesize when measured within
+          #   `limit`, including `0`; `nil` when input is missing, over the
+          #   limit, or cannot be preserved for downstream reads
+          def peek_bytesize(env, limit:)
+            rack_input = env["rack.input"]
+            return unless rack_input
+
+            rewindable = rewind? && rack_input.respond_to?(:rewind)
+
+            # NOTE: Rack 2 requires `rack.input` to be rewindable. Rewind before peeking
+            #       in case an upstream framework already consumed part of the stream
+            return if rewindable && !rewind(rack_input)
+
+            # NOTE: Read one byte past the limit to distinguish an exact-limit body
+            #       from an over-limit body without reading the whole stream.
+            buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
+            over_limit = buffer.bytesize > limit
+
+            if rewindable
+              # NOTE: If we cannot rewind after peeking, downstream code would observe
+              #       a partially consumed body. Treat it as not safely collectable
+              return unless rewind(rack_input)
+            else
+              env["rack.input"] = if over_limit
+                BufferedInput.new(rack_input, buffer: StringIO.new(buffer))
+              else
+                StringIO.new(buffer)
+              end
+            end
+
+            over_limit ? nil : buffer.bytesize
+          end
+
+          private_class_method def rewind?
+            return @rewind if defined?(@rewind)
+
+            @rewind = ::Gem::Version.new(::Rack.release) < ::Gem::Version.new("3")
+          end
+
+          private_class_method def rewind(io)
+            io.rewind
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
+require_relative "../integration"
 
-require_relative 'patcher'
-require_relative 'request_middleware'
-require_relative 'request_body_middleware'
+require_relative "patcher"
+require_relative "request_middleware"
+require_relative "request_body_middleware"
 
 module Datadog
   module AppSec
@@ -14,12 +14,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('1.1.0')
+          MINIMUM_VERSION = Gem::Version.new("1.1.0")
 
           register_as :rack, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['rack']&.version
+            Gem.loaded_specs["rack"]&.version
           end
 
           def self.loaded?
@@ -27,7 +27,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rack/patcher.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../../monitor'
-require_relative 'gateway/watcher'
+require_relative "../../monitor"
+require_relative "gateway/watcher"
 
 module Datadog
   module AppSec
@@ -12,7 +12,7 @@ module Datadog
           module_function
 
           def patched?
-            Patcher.instance_variable_get(:@patched) # TODO: Patcher.flag_patched
+            !!Patcher.instance_variable_get(:@patched) # TODO: Patcher.flag_patched
           end
 
           def target_version

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'gateway/request'
-require_relative '../../instrumentation/gateway'
-require_relative '../../response'
+require_relative "gateway/request"
+require_relative "../../instrumentation/gateway"
+require_relative "../../response"
 
 module Datadog
   module AppSec
@@ -12,7 +12,8 @@ module Datadog
         # This should be inserted just below Rack::JSONBodyParser or
         # legacy Rack::PostBodyContentTypeParser from rack-contrib
         class RequestBodyMiddleware
-          def initialize(app, opt = {})
+          # TODO: opt is never used, it can probably be safely removed
+          def initialize(app, opt = {}) # steep:ignore DifferentMethodParameterKind
             @app = app
           end
 
@@ -23,18 +24,20 @@ module Datadog
 
             # TODO: handle exceptions, except for @app.call
 
-            http_response = nil
+            http_response = nil #: Rack::response?
             interrupt_params = catch(::Datadog::AppSec::Ext::INTERRUPT) do
-              http_response, _request = Instrumentation.gateway.push('rack.request.body', Gateway::Request.new(env)) do
+              http_response, _request = Instrumentation.gateway.push("rack.request.body", Gateway::Request.new(env)) do
                 @app.call(env)
               end
 
               nil
             end
 
-            return AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack if interrupt_params
+            return AppSec::Response.from_interrupt_params(interrupt_params, env["HTTP_ACCEPT"]).to_rack if interrupt_params
 
-            http_response
+            # Steep can't see that the catch block always populates http_response
+            # when no interrupt is thrown.
+            http_response #: Rack::response
           end
         end
       end