datadog 2.36.0 → 2.38.0

data/ext/datadog_profiling_native_extension/collectors_thread_context.h

@@ -21,6 +21,9 @@ VALUE enforce_thread_context_collector_instance(VALUE object);
 void thread_context_collector_stats(VALUE self_instance, VALUE stats_hash);
 void thread_context_collector_stats_reset_not_thread_safe(VALUE self_instance);
 void thread_context_collector_on_serialize(VALUE self_instance);
+void thread_context_collector_reset_all_per_thread_contexts(VALUE self_instance);
+void thread_context_collector_profiler_internal_thread_started(void);
+void thread_context_collector_profiler_internal_thread_done(VALUE self_instance);
 
 #ifndef NO_GVL_INSTRUMENTATION
   typedef enum {

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -83,6 +83,12 @@ Logging.message("[datadog] End of compiler information\n")
 # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
 append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
 
+# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
+# (a header-dedup regression), which `-Werror` turns into fatal
+# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
+# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
+append_cflags "-Wno-error=typedef-redefinition" if ENV["DATADOG_GEM_CI"] == "true"
+
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
 # * by msgpack, another datadog gem dependency
@@ -126,9 +132,6 @@ if RUBY_PLATFORM.include?("linux")
   # but it's slower to build
   # so instead we just assume that we have the function we need on Linux, and nowhere else
   $defs << "-DHAVE_PTHREAD_GETCPUCLOCKID"
-
-  # Not available on macOS
-  $defs << "-DHAVE_CLOCK_MONOTONIC_COARSE"
 elsif RUBY_PLATFORM.include?("darwin")
   # On macOS, we use Mach thread APIs to get per-thread CPU time
   $defs << "-DHAVE_MACH_THREAD_INFO"
@@ -154,6 +157,11 @@ $defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
 # (see https://bugs.ruby-lang.org/issues/21710)
 $defs << "-DUSE_DEFERRED_HEAP_ALLOCATION_RECORDING" unless RUBY_VERSION < "4"
 
+# On Ruby 4.0, we've seen crashes when computing the memsize of a class/module/iclass:
+# rb_obj_memsize_of walks the per-namespace class extensions (classext_memsize), which seem to sometimes be in an inconsistent state
+# (see https://github.com/DataDog/dd-trace-rb/issues/5936)
+$defs << "-DNO_SAFE_CLASS_MEMSIZE" unless RUBY_VERSION < "4"
+
 # This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
 # It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
 # actually required for correct behavior of the profiler.

data/ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -871,3 +871,8 @@ bool is_raised_flag_set(VALUE thread) { return thread_struct_from_object(thread)
   VALUE current_fiber_for(DDTRACE_UNUSED VALUE thread) { rb_raise(rb_eRuntimeError, "Not implemented for Ruby < 3.1"); }
   void self_test_current_fiber_for(void) { } // Nothing to do
 #endif
+
+bool pathobj_is_null(VALUE iseq) {
+  const rb_iseq_t *iseq_ptr = (const rb_iseq_t *) iseq;
+  return ISEQ_BODY(iseq_ptr)->location.pathobj == 0;
+}

data/ext/datadog_profiling_native_extension/private_vm_api_access.h

@@ -83,3 +83,5 @@ bool is_raised_flag_set(VALUE thread);
 VALUE current_fiber_for(VALUE thread);
 
 void self_test_current_fiber_for(void);
+
+bool pathobj_is_null(VALUE iseq);

data/ext/datadog_profiling_native_extension/ruby_helpers.c

@@ -162,9 +162,16 @@ size_t rb_obj_memsize_of(VALUE obj);
 size_t ruby_obj_memsize_of(VALUE obj) {
   switch (rb_type(obj)) {
     case T_OBJECT:
+    // On Ruby 4.0, computing the size of a class/module/iclass seems to not be safe: `rb_obj_memsize_of` walks the
+    // per-namespace class extensions (`rb_class_classext_foreach` -> `classext_memsize` -> `rb_id_table_memsize`)
+    // and can crash the VM when called on objects tracked by heap profiling.
+    // See https://github.com/DataDog/dd-trace-rb/issues/5936. Class objects contribute negligibly to heap size,
+    // so we skip them (fall through to the `default` branch, returning 0) rather than risk a SIGSEGV.
+    #ifndef NO_SAFE_CLASS_MEMSIZE
     case T_MODULE:
     case T_CLASS:
     case T_ICLASS:
+    #endif
     case T_STRING:
     case T_ARRAY:
     case T_HASH:

data/ext/datadog_profiling_native_extension/time_helpers.h

@@ -4,7 +4,7 @@
 #include <errno.h>
 #include <time.h>
 
-#include "extconf.h" // Needed for HAVE_CLOCK_MONOTONIC_COARSE
+#include "extconf.h"
 #include "ruby_helpers.h"
 
 #define SECONDS_AS_NS(value) (value * 1000 * 1000 * 1000L)
@@ -33,7 +33,16 @@ static inline long retrieve_clock_as_ns(clockid_t clock_id, raise_on_failure_set
   return clock_value.tv_nsec + SECONDS_AS_NS(clock_value.tv_sec);
 }
 
-static inline long monotonic_wall_time_now_ns(raise_on_failure_setting raise_on_failure) { return retrieve_clock_as_ns(CLOCK_MONOTONIC, raise_on_failure); }
+// CLOCK_MONOTONIC on macOS only has microsecond precision, CLOCK_MONOTONIC_RAW has nanosecond precision
+#ifdef __APPLE__
+  #define CLOCK_MONOTONIC_FOR_PROFILING CLOCK_MONOTONIC_RAW
+  #define CLOCK_MONOTONIC_COARSE_FOR_PROFILING CLOCK_MONOTONIC_RAW_APPROX
+#else
+  #define CLOCK_MONOTONIC_FOR_PROFILING CLOCK_MONOTONIC
+  #define CLOCK_MONOTONIC_COARSE_FOR_PROFILING CLOCK_MONOTONIC_COARSE
+#endif
+
+static inline long monotonic_wall_time_now_ns(raise_on_failure_setting raise_on_failure) { return retrieve_clock_as_ns(CLOCK_MONOTONIC_FOR_PROFILING, raise_on_failure); }
 static inline long system_epoch_time_now_ns(raise_on_failure_setting raise_on_failure)   { return retrieve_clock_as_ns(CLOCK_REALTIME,  raise_on_failure); }
 
 // Coarse instants use CLOCK_MONOTONIC_COARSE on Linux which is expected to provide resolution in the millisecond range:
@@ -47,11 +56,7 @@ typedef struct {
 static inline coarse_instant to_coarse_instant(long timestamp_ns) { return (coarse_instant) {.timestamp_ns = timestamp_ns}; }
 
 static inline coarse_instant monotonic_coarse_wall_time_now_ns(void) {
- #ifdef HAVE_CLOCK_MONOTONIC_COARSE // Linux
-    return to_coarse_instant(retrieve_clock_as_ns(CLOCK_MONOTONIC_COARSE, DO_NOT_RAISE_ON_FAILURE));
-  #else // macOS
-    return to_coarse_instant(retrieve_clock_as_ns(CLOCK_MONOTONIC, DO_NOT_RAISE_ON_FAILURE));
-  #endif
+  return to_coarse_instant(retrieve_clock_as_ns(CLOCK_MONOTONIC_COARSE_FOR_PROFILING, DO_NOT_RAISE_ON_FAILURE));
 }
 
 long monotonic_to_system_epoch_ns(monotonic_to_system_epoch_state *state, long monotonic_wall_time_ns);

data/ext/libdatadog_api/di.c

@@ -21,6 +21,12 @@ void rb_objspace_each_objects(
 // from standard library exception classes like NameError.
 static ID id_mesg;
 
+// ID for the fiber-local key that backs the method-probe re-entrancy guard.
+// Storage is the same hashtable that backs Thread#[] / Thread#[]=, but accessed
+// directly via rb_thread_local_aref / rb_thread_local_aset so that user-installed
+// method probes on Thread#[] / Thread#[]= cannot intercept guard reads/writes.
+static ID id_datadog_di_in_probe;
+
 // Returns whether the argument is an IMEMO of type ISEQ.
 static bool ddtrace_imemo_iseq_p(VALUE v) {
   return rb_objspace_internal_object_p(v) && RB_TYPE_P(v, T_IMEMO) && ddtrace_imemo_type(v) == IMEMO_TYPE_ISEQ;
@@ -75,6 +81,71 @@ static VALUE exception_message(DDTRACE_UNUSED VALUE _self, VALUE exception) {
   return rb_ivar_get(exception, id_mesg);
 }
 
+/*
+ * call-seq:
+ *   DI.in_probe? -> true | false
+ *
+ * Returns whether the current fiber is currently inside DI probe processing.
+ * Reads the same fiber-local storage as Thread.current[:datadog_di_in_probe]
+ * but bypasses Thread#[] method dispatch — a user method probe on Thread#[]
+ * cannot observe or intercept this call.
+ *
+ * @api private
+ */
+static VALUE in_probe_p(DDTRACE_UNUSED VALUE _self) {
+  VALUE v = rb_thread_local_aref(rb_thread_current(), id_datadog_di_in_probe);
+  return RTEST(v) ? Qtrue : Qfalse;
+}
+
+/*
+ * call-seq:
+ *   DI.enter_probe -> nil
+ *
+ * Marks the current fiber as inside DI probe processing. Writes to the same
+ * fiber-local storage as Thread.current[:datadog_di_in_probe] = true, but
+ * bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
+ * cannot observe or intercept this call.
+ *
+ * @api private
+ */
+static VALUE enter_probe(DDTRACE_UNUSED VALUE _self) {
+  rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qtrue);
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *   DI.leave_probe -> nil
+ *
+ * Marks the current fiber as no longer inside DI probe processing. Writes to
+ * the same fiber-local storage as Thread.current[:datadog_di_in_probe] = nil,
+ * but bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
+ * cannot observe or intercept this call.
+ *
+ * @api private
+ */
+static VALUE leave_probe(DDTRACE_UNUSED VALUE _self) {
+  rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qnil);
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *   DI.hash?(obj) -> true | false
+ *
+ * Returns whether the given object is a Hash via a direct type check
+ * (RB_TYPE_P), bypassing Kernel#is_a? method dispatch. Used in the method
+ * probe wrapper (the Ruby < 3 argument-serialization split) to test whether
+ * the trailing argument is a hash without giving a user-installed method
+ * probe on Kernel#is_a? a chance to recurse. Matches Hash and its
+ * subclasses, like is_a?(Hash), since their instances are still T_HASH.
+ *
+ * @api private
+ */
+static VALUE is_hash(DDTRACE_UNUSED VALUE _self, VALUE obj) {
+  return RB_TYPE_P(obj, T_HASH) ? Qtrue : Qfalse;
+}
+
 // rb_iseq_type was added in Ruby 3.1 (commit 89a02d89 by Koichi Sasada,
 // 2021-12-19). It returns the iseq type as a Symbol. On Ruby < 3.1 this
 // function does not exist, so have_func('rb_iseq_type') in extconf.rb
@@ -117,10 +188,15 @@ static VALUE iseq_type(DDTRACE_UNUSED VALUE _self, VALUE iseq_val) {
 
 void di_init(VALUE datadog_module) {
   id_mesg = rb_intern("mesg");
+  id_datadog_di_in_probe = rb_intern("datadog_di_in_probe");
 
   VALUE di_module = rb_define_module_under(datadog_module, "DI");
   rb_define_singleton_method(di_module, "all_iseqs", all_iseqs, 0);
   rb_define_singleton_method(di_module, "exception_message", exception_message, 1);
+  rb_define_singleton_method(di_module, "in_probe?", in_probe_p, 0);
+  rb_define_singleton_method(di_module, "enter_probe", enter_probe, 0);
+  rb_define_singleton_method(di_module, "leave_probe", leave_probe, 0);
+  rb_define_singleton_method(di_module, "hash?", is_hash, 1);
 #ifdef HAVE_RB_ISEQ_TYPE
   rb_define_singleton_method(di_module, "iseq_type", iseq_type, 1);
 #endif

data/ext/libdatadog_api/extconf.rb

@@ -41,6 +41,12 @@ Datadog::LibdatadogExtconfHelpers.dump_mkmf_log_on_failure!
 # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
 append_cflags '-Werror' if ENV['DATADOG_GEM_CI'] == 'true'
 
+# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
+# (a header-dedup regression), which `-Werror` turns into fatal
+# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
+# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
+append_cflags '-Wno-error=typedef-redefinition' if ENV['DATADOG_GEM_CI'] == 'true'
+
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
 # * by msgpack, another datadog gem dependency

data/ext/libdatadog_extconf_helpers.rb

@@ -10,7 +10,7 @@ module Datadog
   module LibdatadogExtconfHelpers
     # Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
     # may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
-    LIBDATADOG_VERSION = '~> 35.0.0.1.0'
+    LIBDATADOG_VERSION = '~> 36.0.0.1.0'
 
     # Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
     # libdatadog are moved after the extension gets compiled.

data/lib/datadog/ai_guard/api_client.rb

@@ -17,11 +17,13 @@ module Datadog
 
         @endpoint_uri = if endpoint
           URI(endpoint) #: URI::HTTP
+        elsif Datadog.configuration.site
+          host = Datadog.configuration.site.dup
+          host.prepend("app.") if host.count(".") == 1
+
+          URI::HTTPS.build(host: host, path: DEFAULT_PATH)
         else
-          URI::HTTPS.build(
-            host: Datadog.configuration.site || DEFAULT_SITE,
-            path: DEFAULT_PATH
-          )
+          URI::HTTPS.build(host: DEFAULT_SITE, path: DEFAULT_PATH)
         end
 
         @headers = {

data/lib/datadog/appsec/api_security/route_extractor.rb

@@ -31,16 +31,19 @@ module Datadog
         #         uses `sinatra.route` with a string like "GET /users/:id"
         #       Grape
         #         uses `grape.routing_args` with a hash with a `:route_info` key
-        #         that contains a `Grape::Router::Route` object that contains
-        #         `Grape::Router::Pattern` object with an `origin` method
+        #         that contains a {Grape::Router::Route} object that contains
+        #         {Grape::Router::Pattern} object with an `origin` method
+        #       Rails with `action_pack` tracing contrib (fast path)
+        #         `datadog.action_dispatch.route` stores the {Journey::Route}
+        #         object set by the tracer at routing time
         #       Rails < 7.1 (slow path)
-        #         uses `action_dispatch.routes` to store `ActionDispatch::Routing::RouteSet`
+        #         uses `action_dispatch.routes` to store {ActionDispatch::Routing::RouteSet}
         #         which can recognize requests
         #       Rails > 7.1 (fast path)
         #         uses `action_dispatch.route_uri_pattern` with a string like
         #         "/users/:id(.:format)"
         #       Rails > 8.1.1 (fast path)
-        #         uses `action_dispatch.route` to store the ActionDispatch::Journey::Route
+        #         uses `action_dispatch.route` to store the {ActionDispatch::Journey::Route}
         #         that matched when the request was routed
         #
         # WARNING: This method works only *after* the request has been routed.

data/lib/datadog/appsec/configuration.rb

@@ -414,6 +414,12 @@ module Datadog
                     o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS'
                     o.default 1
                   end
+
+                  option :max_downstream_body_bytes do |o|
+                    o.type :int
+                    o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES'
+                    o.default 10_485_760
+                  end
                 end
               end
 

data/lib/datadog/appsec/context.rb

@@ -20,7 +20,7 @@ module Datadog
       ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
 
       # TODO: add delegators for active trace span
-      attr_reader :trace, :span
+      attr_reader :trace, :span, :metrics
 
       # Shared mutable storage for counters, flags, and data accumulated during
       # the request's lifecycle.
@@ -103,7 +103,8 @@ module Datadog
       end
 
       def extract_schema!
-        waf_result = @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
+        persistent_data = {'waf.context.processor' => {'extract-schema' => true}}
+        waf_result = run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
         security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
 
         @state[:schema_extracted] = security_event.schema?
@@ -116,6 +117,7 @@ module Datadog
 
         Metrics::Exporter.export_waf_metrics(@metrics.waf, @span)
         Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
+        Metrics::Exporter.export_downstream_response_metrics(@metrics.downstream_responses, @span)
       end
 
       def export_request_telemetry

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -7,6 +7,8 @@ require_relative '../../trace_keeper'
 require_relative '../../security_event'
 require_relative '../../utils/http/url_encoded'
 require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -39,7 +41,7 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data[:body], content_type: headers['content-type'])
+              body = parse_request_body(data[:body], content_type: headers['content-type'])
               ephemeral_data['server.io.net.request.body'] = body if body
             end
 
@@ -67,7 +69,7 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
               ephemeral_data['server.io.net.response.body'] = body if body
             end
 
@@ -89,11 +91,64 @@ module Datadog
             data[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def request_url(data)

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -3,8 +3,9 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
-require_relative '../../utils/http/media_type'
 require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -37,7 +38,7 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
+              body = parse_request_body(env.body, content_type: headers['content-type'])
               ephemeral_data['server.io.net.request.body'] = body if body
             end
 
@@ -64,7 +65,7 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
+              body = parse_response_body(env.body, headers: headers, context: context)
               ephemeral_data['server.io.net.response.body'] = body if body
             end
 
@@ -82,11 +83,64 @@ module Datadog
             env[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def normalize_headers(headers)

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -72,11 +72,11 @@ module Datadog
                     args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
                   )
 
+                  # re-used fragments are not expanded
                   visited_fragments[fragment_name] = true
                   arguments_from_selections(
                     fragment.selections, query_variables, args_hash, fragments, visited_fragments
                   )
-                  visited_fragments.delete(fragment_name)
                 when ::GraphQL::Language::Nodes::Field
                   selection_name = selection.alias || selection.name
                   field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -83,6 +83,11 @@ module Datadog
               # usually Hash[String, String] but can be a more complex
               # Hash[String, (String|Array|Hash)] when e.g coming from JSON
               env['rack.request.form_hash']
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request body: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request body')
+
+              nil
             end
 
             # Returns the request body size in bytes using all available methods,

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -104,6 +104,10 @@ module Datadog
                       body = gateway_request.form_hash
                       persistent_data['server.request.body'] = body if body
                     end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?('rack.request.form_hash')
+                    body = gateway_request.env['rack.request.form_hash']
+                    persistent_data['server.request.body'] = body if body
                   end
 
                   next stack.call(request) if persistent_data.empty?

data/lib/datadog/appsec/contrib/rack/input_peeker.rb

@@ -2,6 +2,7 @@
 
 require 'stringio'
 require_relative 'buffered_input'
+require_relative '../../utils/http/body_reader'
 
 module Datadog
   module AppSec
@@ -38,7 +39,9 @@ module Datadog
             #       in case an upstream framework already consumed part of the stream
             return if rewindable && !rewind(rack_input)
 
-            buffer = peek(rack_input, limit)
+            # NOTE: Read one byte past the limit to distinguish an exact-limit body
+            #       from an over-limit body without reading the whole stream.
+            buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
             over_limit = buffer.bytesize > limit
 
             if rewindable
@@ -69,22 +72,6 @@ module Datadog
             Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
             false
           end
-
-          private_class_method def peek(io, limit)
-            # NOTE: Read one byte past the limit to distinguish an exact-limit body
-            #       from an over-limit body without reading the whole stream.
-            max = limit + 1
-            buffer = +''.b
-
-            while buffer.bytesize <= limit
-              chunk = io.read(max - buffer.bytesize)
-              break if chunk.nil? || chunk.empty?
-
-              buffer << chunk
-            end
-
-            buffer
-          end
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -10,6 +10,7 @@ require_relative '../../event'
 require_relative '../../response'
 require_relative '../../api_security'
 require_relative '../../default_header_tags'
+require_relative '../../route_normalizer'
 require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
@@ -76,6 +77,8 @@ module Datadog
               nil
             end
 
+            add_normalized_route_tag(ctx, env)
+
             if interrupt_params
               ctx.mark_as_interrupted!
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
@@ -198,6 +201,24 @@ module Datadog
             end
           end
 
+          def add_normalized_route_tag(context, env)
+            return unless AppSec::APISecurity.enabled?
+
+            span = context.span
+            return unless span
+
+            pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
+            return unless pattern
+
+            # NOTE: To build full path that covers mounted engines we need to add
+            #       pre-computed by Tracer route path tag to the normalized route
+            prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env['SCRIPT_NAME']
+            normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
+            return unless normalized_route
+
+            span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
+          end
+
           def oneshot_tags_sent?
             @oneshot_tags_sent
           end