datadog 2.8.0 → 2.39.0

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require_relative 'gateway/multiplex'
-require_relative '../../instrumentation/gateway'
+require "json"
+require_relative "gateway/multiplex"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -16,16 +16,10 @@ module Datadog
 
             gateway_multiplex = Gateway::Multiplex.new(multiplex)
 
-            multiplex_return, multiplex_response = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push("graphql.multiplex", gateway_multiplex) do
               super
             end
 
-            # Returns an error * the number of queries so that the entire multiplex is blocked
-            if multiplex_response
-              blocked_event = multiplex_response.find { |action, _options| action == :block }
-              multiplex_return = AppSec::Response.graphql_response(gateway_multiplex) if blocked_event
-            end
-
             multiplex_return
           end
         end

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'graphql'
+require "graphql"
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec
@@ -47,40 +47,86 @@ module Datadog
             # Note that the `comments` "include" directive is included in the arguments list
             def build_arguments_hash
               queries.each_with_object({}) do |query, args_hash|
-                next unless query.selected_operation
+                selected_operation = query.selected_operation
+                next unless selected_operation
 
-                arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
+                arguments_from_selections(selected_operation.selections, query.variables, args_hash, query.fragments)
               end
             end
 
-            def arguments_from_selections(selections, query_variables, args_hash)
+            def arguments_from_selections(selections, query_variables, args_hash, fragments, visited_fragments = {})
               selections.each do |selection|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless selection.class.name == Integration::AST_NODE_CLASS_NAMES[:field]
-                # rubocop:enable Style/ClassEqualityComparison
-
-                selection_name = selection.alias || selection.name
-
-                if !selection.arguments.empty? || !selection.directives.empty?
-                  args_hash[selection_name] ||= []
-                  args_hash[selection_name] <<
-                    arguments_hash(selection.arguments, query_variables).merge!(
-                      arguments_from_directives(selection.directives, query_variables)
-                    )
+                case selection
+                when ::GraphQL::Language::Nodes::FragmentSpread
+                  fragment_name = selection.name
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  next if visited_fragments[fragment_name]
+
+                  fragment = fragments[fragment_name]
+                  next unless fragment
+
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
+                  )
+
+                  # re-used fragments are not expanded
+                  visited_fragments[fragment_name] = true
+                  arguments_from_selections(
+                    fragment.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::Field
+                  selection_name = selection.alias || selection.name
+                  field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
+                  append_arguments(
+                    args_hash,
+                    selection_name,
+                    field_arguments,
+                    arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::InlineFragment
+                  append_arguments(
+                    args_hash, selection.type.name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
                 end
+              end
+            end
 
-                arguments_from_selections(selection.selections, query_variables, args_hash)
+            def append_arguments(args_hash, selection_name, arguments, directive_arguments)
+              combined_arguments = if arguments
+                arguments.merge!(directive_arguments) if directive_arguments
+                arguments
+              else
+                directive_arguments
               end
+              return unless combined_arguments
+
+              args_hash[selection_name] ||= []
+              args_hash[selection_name] << combined_arguments
             end
 
             def arguments_from_directives(directives, query_variables)
-              directives.each_with_object({}) do |directive, args_hash|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless directive.class.name == Integration::AST_NODE_CLASS_NAMES[:directive]
-                # rubocop:enable Style/ClassEqualityComparison
+              return if directives.empty?
+
+              directive_arguments = directives.each_with_object({}) do |directive, args_hash|
+                next unless directive.is_a?(::GraphQL::Language::Nodes::Directive)
 
                 args_hash[directive.name] = arguments_hash(directive.arguments, query_variables)
               end
+
+              return if directive_arguments.empty?
+
+              directive_arguments
             end
 
             def arguments_hash(arguments, query_variables)
@@ -90,15 +136,19 @@ module Datadog
             end
 
             def argument_value(argument, query_variables)
-              case argument.value.class.name
+              value = argument.value
+
+              case value.class.name
               when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
+                # @type var value: GraphQL::Language::Nodes::VariableIdentifier
                 # we need to pass query.variables here instead of query.provided_variables,
                 # since #provided_variables don't know anything about variable default value
-                query_variables[argument.value.name]
+                query_variables[value.name]
               when Integration::AST_NODE_CLASS_NAMES[:input_object]
-                arguments_hash(argument.value.arguments, query_variables)
+                # @type var value: GraphQL::Language::Nodes::InputObject
+                arguments_hash(value.arguments, query_variables)
               else
-                argument.value
+                value
               end
             end
           end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
-require 'json'
-require_relative '../../../instrumentation/gateway'
-require_relative '../reactive/multiplex'
-require_relative '../../../reactive/operation'
+require "json"
+
+require_relative "../../../event"
+require_relative "../../../trace_keeper"
+require_relative "../../../security_event"
+require_relative "../../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -19,43 +21,30 @@ module Datadog
                 watch_multiplex(gateway)
               end
 
-              # This time we don't throw but use next
               def watch_multiplex(gateway = Instrumentation.gateway)
-                gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
-                  block = false
-                  event = nil
-
-                  scope = AppSec::Scope.active_scope
-
-                  if scope
-                    AppSec::Reactive::Operation.new('graphql.multiplex') do |op|
-                      GraphQL::Reactive::Multiplex.subscribe(op, scope.processor_context) do |result|
-                        event = {
-                          waf_result: result,
-                          trace: scope.trace,
-                          span: scope.service_entry_span,
-                          multiplex: gateway_multiplex,
-                          actions: result.actions
-                        }
+                gateway.watch("graphql.multiplex") do |stack, gateway_multiplex|
+                  context = AppSec::Context.active
 
-                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
-                        scope.processor_context.events << event
-                      end
+                  if context
+                    persistent_data = {
+                      "graphql.server.all_resolvers" => gateway_multiplex.arguments
+                    }
 
-                      block = GraphQL::Reactive::Multiplex.publish(op, gateway_multiplex)
-                    end
-                  end
+                    result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  next [nil, [[:block, event]]] if block
+                    if result.match?
+                      AppSec::Event.tag(context, result)
+                      TraceKeeper.keep!(context.trace) if result.keep?
 
-                  ret, res = stack.call(gateway_multiplex.arguments)
+                      context.events.push(
+                        AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                      )
 
-                  if event
-                    res ||= []
-                    res << [:monitor, event]
+                      AppSec::ActionsHandler.handle(result.actions)
+                    end
                   end
 
-                  [ret, res]
+                  stack.call(gateway_multiplex.arguments)
                 end
               end
             end

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
-require_relative 'patcher'
+require_relative "../integration"
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -11,19 +11,20 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('2.0.19')
+          MINIMUM_VERSION = Gem::Version.new("2.0.19")
 
           AST_NODE_CLASS_NAMES = {
-            field: 'GraphQL::Language::Nodes::Field',
-            directive: 'GraphQL::Language::Nodes::Directive',
-            variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
-            input_object: 'GraphQL::Language::Nodes::InputObject',
+            field: "GraphQL::Language::Nodes::Field",
+            directive: "GraphQL::Language::Nodes::Directive",
+            fragment_spread: "GraphQL::Language::Nodes::FragmentSpread",
+            variable_identifier: "GraphQL::Language::Nodes::VariableIdentifier",
+            input_object: "GraphQL::Language::Nodes::InputObject",
           }.freeze
 
           register_as :graphql, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version
+            Gem.loaded_specs["graphql"]&.version
           end
 
           def self.loaded?
@@ -31,7 +32,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION && ast_node_classes_defined?
+            !!(super && version&.>=(MINIMUM_VERSION) && ast_node_classes_defined?)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/graphql/patcher.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../patcher'
-require_relative 'gateway/watcher'
+require_relative "gateway/watcher"
 
-if Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version >= Gem::Version.new('2.0.19')
-  require_relative 'appsec_trace'
+if Gem.loaded_specs["graphql"] && Gem.loaded_specs["graphql"].version >= Gem::Version.new("2.0.19")
+  require_relative "appsec_trace"
 end
 
 module Datadog
@@ -13,8 +12,6 @@ module Datadog
       module GraphQL
         # Patcher for AppSec on GraphQL
         module Patcher
-          include Datadog::AppSec::Contrib::Patcher
-
           module_function
 
           def patched?

data/lib/datadog/appsec/contrib/rack/buffered_input.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Wraps a `rack.input` stream with a buffer placed in front of it.
+        # Every read drains the buffer first, then continues from the stream
+        #
+        # NOTE: Forward-only: no rewind, no seek
+        #
+        # NOTE: Rack 3 dropped the rewind requirement from the input stream contract
+        # @see https://github.com/rack/rack/blob/v3.2.6/SPEC.rdoc
+        class BufferedInput
+          # NOTE: Rack's multipart parser reads in 1 MiB chunks, used to bound
+          #       {#each} the same way
+          # @see https://github.com/rack/rack/blob/v3.2.6/lib/rack/multipart/parser.rb#L54
+          READ_BUFSIZE_BYTES = 1_048_576
+
+          def initialize(stream, buffer:)
+            @stream = stream
+            @buffer = buffer
+          end
+
+          def read(length = nil, outbuf = nil)
+            if length.nil?
+              data = @buffer.read(nil, outbuf) || +""
+              more = @stream.read
+
+              data << more if more
+
+              return data
+            end
+
+            data = @buffer.read(length, outbuf)
+
+            if data.nil?
+              more = @stream.read(length, outbuf)
+              return more if more && !more.empty?
+
+              # NOTE: Match `IO#read(length, outbuf)` at EOF. Return nil and clear
+              #       the caller's buffer so stale bytes are not mistaken for data
+              outbuf&.clear
+              return
+            end
+
+            remaining = length - data.bytesize
+            return data if remaining <= 0
+
+            more = @stream.read(remaining)
+            data << more if more
+
+            data
+          end
+
+          def gets
+            line = @buffer.gets
+
+            return @stream.gets if line.nil?
+            return line if line.end_with?("\n")
+
+            more = @stream.gets
+            more ? (line << more) : line
+          end
+
+          def each
+            while (chunk = read(READ_BUFSIZE_BYTES))
+              yield chunk
+            end
+
+            self
+          end
+
+          def close
+            @buffer.close
+          ensure
+            @stream.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -6,6 +6,40 @@ module Datadog
       module Rack
         # Rack integration constants
         module Ext
+          COLLECTABLE_REQUEST_HEADERS = [
+            "accept",
+            "akamai-user-risk",
+            "cf-ray",
+            "cloudfront-viewer-ja3-fingerprint",
+            "content-type",
+            "user-agent",
+            "x-amzn-trace-id",
+            "x-appgw-trace-id",
+            "x-cloud-trace-context",
+            "x-sigsci-requestid",
+            "x-sigsci-tags"
+          ].freeze
+
+          IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
+            "accept-encoding",
+            "accept-language",
+            "cf-connecting-ip",
+            "cf-connecting-ipv6",
+            "content-encoding",
+            "content-language",
+            "content-length",
+            "fastly-client-ip",
+            "forwarded",
+            "forwarded-for",
+            "host",
+            "true-client-ip",
+            "via",
+            "x-client-ip",
+            "x-cluster-client-ip",
+            "x-forwarded",
+            "x-forwarded-for",
+            "x-real-ip"
+          ].freeze
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
-require_relative '../../../../core/header_collection'
-require_relative '../../../../tracing/client_ip'
+require_relative "../input_peeker"
+require_relative "../../../instrumentation/gateway/argument"
+require_relative "../../../utils/http/url_encoded"
+require_relative "../../../../core/header_collection"
+require_relative "../../../../tracing/client_ip"
 
 module Datadog
   module AppSec
@@ -23,16 +25,7 @@ module Datadog
             end
 
             def query
-              # Downstream libddwaf expects keys and values to be extractable
-              # separately so we can't use [[k, v], ...]. We also want to allow
-              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
-              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
-              request.query_string.split('&').each.with_object({}) do |e, hash|
-                k, v = e.split('=').map { |s| CGI.unescape(s) }
-                hash[k] ||= []
-
-                hash[k] << v
-              end
+              Utils::HTTP::URLEncoded.parse(request.query_string)
             end
 
             def method
@@ -41,12 +34,12 @@ module Datadog
 
             def headers
               result = request.env.each_with_object({}) do |(k, v), h|
-                h[k.delete_prefix('HTTP_').tap(&:downcase!).tap { |s| s.tr!('_', '-') }] = v if k.start_with?('HTTP_')
+                h[k.delete_prefix("HTTP_").tap(&:downcase!).tap { |s| s.tr!("_", "-") }] = v if k.start_with?("HTTP_")
               end
 
-              result['content-type'] = request.content_type if request.content_type
+              result["content-type"] = request.content_type if request.content_type
               # Since Rack 3.1, content-length is nil if the body is empty, but we still want to send it to the WAF.
-              result['content-length'] = request.content_length || '0'
+              result["content-length"] = request.content_length || "0"
               result
             end
 
@@ -75,16 +68,58 @@ module Datadog
             end
 
             def remote_addr
-              env['REMOTE_ADDR']
+              env["REMOTE_ADDR"]
             end
 
             def form_hash
-              # force form data processing
+              # NOTE: Rack populates `rack.request.form_hash` only as a side effect
+              #       of {::Rack::Request#POST}, which reads and parses the body.
               request.POST if request.form_data?
 
-              # usually Hash<String,String> but can be a more complex
-              # Hash<String,String||Array||Hash> when e.g coming from JSON
-              env['rack.request.form_hash']
+              # usually Hash[String, String] but can be a more complex
+              # Hash[String, (String|Array|Hash)] when e.g coming from JSON
+              env["rack.request.form_hash"]
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request body: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: "AppSec: Failed to parse request body")
+
+              nil
+            end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       size if it's known, content-length if provided, and buffering
+            #       to the limit if unknown-length
+            #
+            # WARNING: The buffering path adds overhead for streaming web-servers
+            #          (Rack 3+) when the body length is unknown
+            def body_bytesize(limit)
+              io = request.body
+
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: {::Rack::Request#content_length} is a plain `CONTENT_LENGTH`
+              #       getter forces no body read
+              content_length = request.content_length
+              return content_length.to_i if content_length
+
+              # NOTE: A parsed-body cache without a raw byte count means the input
+              #       stream was consumed before measurement, consider it unknown-length
+              return if env.key?("rack.request.form_hash")
+
+              InputPeeker.peek_bytesize(env, limit: limit)
+            end
+
+            # Whether a request body can be collected without forcing a parse:
+            # either form data parseable on demand, or a body already parsed upstream
+            #
+            # NOTE: Rack does not parse JSON itself, a body parser middleware such as
+            #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
+            def collectable_body?
+              request.form_data? || request.parseable_data? || env.key?("rack.request.form_hash")
             end
 
             def client_ip

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec
@@ -9,14 +9,14 @@ module Datadog
         module Gateway
           # Gateway Response argument.
           class Response < Instrumentation::Gateway::Argument
-            attr_reader :body, :status, :headers, :scope
+            attr_reader :body, :status, :headers, :context
 
-            def initialize(body, status, headers, scope:)
+            def initialize(body, status, headers, context:)
               super()
               @body = body
               @status = status
               @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
-              @scope = scope
+              @context = context
             end
 
             def response