ddtrace 1.11.1 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fab36ba8f93f2c9b067c2fcb4916d6aff5ef906d6608671f035490f3b287c6c9
-  data.tar.gz: 0f62cd63c95aa669f8981cb79c56e49228351a01920c142a36170a9443b60400
+  metadata.gz: 59174b63b257f636df60be99956c8e221fcf046ace610c57eb01192fac6560ad
+  data.tar.gz: 647e8de04a54a65b40ad7b20061a015f510966f7643c192df30f3710a8335386
 SHA512:
-  metadata.gz: 6a747def8ca5cef1d31d72fff939567669c72db53b8c30a3b7cd14df8b52da21a4f5d18efbeba1a22686196923ed49195194b7a9136840428d73f4f8e985da67
-  data.tar.gz: 72f36c83fac54b4d503d38f40759a89d400c5d70b24fbcb7ebc8a291d603c0bab07f28ca58082b732afb66960fe9d6e7ab7148147f0e1b727bbaeb4f08a885a6
+  metadata.gz: b677b76e9e3b00df5e9c5b33bfe8c6007cd867053366364769391bb1b41457e9e380c5869bb85cc97071969e4e6164fcbe89148aaac11828bdc9de7957cc5d21
+  data.tar.gz: 33d66238def0a52a491175d015bfe80e15b84e975abc974710d8b892fb76ffe4ff336a15f23341ecd94b5979d7015d75dc93587d7e227d0bd759c6fdb5c782c4

data/CHANGELOG.md

@@ -2,6 +2,34 @@
 
 ## [Unreleased]
 
+## [1.12.0] - 2023-06-02
+
+### Added
+* Profiling: Add support for profiling Ruby 3.3.0-preview1 ([#2860][])
+* Appsec: Appsec support nested apps ([#2836][])
+* Appsec: Appsec add support for custom rules ([#2856][])
+* Appsec: Update appsec static rules to 1.7.0 version ([#2869][])
+* Appsec: Tag AppSec on Rack top-level span ([#2858][])
+* Profiling: Implement "no signals" workaround and enable CPU Profiling 2.0 for all customers ([#2873][])
+* Ci: Update CI Visibility spec ([#2874][])
+* Appsec: Added missing waf addresses to request operation ([#2883][])
+
+### Changed
+
+* Tracing: Consistent APM Span tags for AWS SDK Requests ([#2730][])
+* Tracing: Change default `service_name` values Part 2 ([#2765][])
+* Profiling: Bump debase-ruby_core_source dependency to 3.2.1 ([#2875][])
+
+### Fixed
+* Telemetry: Disable for non-HTTP agent connection ([#2815][])
+* Tracing: Fix circular requires ([#2826][])
+* Tracing: Update comment about Datadog::Tracing::Distributed::Ext to correct modules ([#2840][])
+* Appsec: Check if `:appsec` setting is present before accessing it in remote component ([#2854][])
+* Telemetry: Do not send Dependency `hash` when `version` is present ([#2855][])
+* Core: Fix symbol configuration for `env` and `service` ([#2864][])
+* Tracing: Fix sql comment propagation `full` mode when tracing is disabled ([#2866][])
+* Appsec: Use relative URI for server.request.uri.raw ([#2890][])
+
 ## [1.11.1] - 2023-05-03
 
 ### Fixed
@@ -2410,7 +2438,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.11.1...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.12.0...master
+[1.12.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.11.1...v1.12.0
 [1.11.1]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.1...v1.11.1
 [1.11.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.1...v1.11.0
 [1.11.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.1...v1.11.0.beta1
@@ -3445,6 +3474,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2720]: https://github.com/DataDog/dd-trace-rb/issues/2720
 [#2726]: https://github.com/DataDog/dd-trace-rb/issues/2726
 [#2727]: https://github.com/DataDog/dd-trace-rb/issues/2727
+[#2730]: https://github.com/DataDog/dd-trace-rb/issues/2730
 [#2731]: https://github.com/DataDog/dd-trace-rb/issues/2731
 [#2732]: https://github.com/DataDog/dd-trace-rb/issues/2732
 [#2733]: https://github.com/DataDog/dd-trace-rb/issues/2733
@@ -3455,6 +3485,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2757]: https://github.com/DataDog/dd-trace-rb/issues/2757
 [#2760]: https://github.com/DataDog/dd-trace-rb/issues/2760
 [#2762]: https://github.com/DataDog/dd-trace-rb/issues/2762
+[#2765]: https://github.com/DataDog/dd-trace-rb/issues/2765
 [#2769]: https://github.com/DataDog/dd-trace-rb/issues/2769
 [#2770]: https://github.com/DataDog/dd-trace-rb/issues/2770
 [#2771]: https://github.com/DataDog/dd-trace-rb/issues/2771
@@ -3466,9 +3497,26 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2805]: https://github.com/DataDog/dd-trace-rb/issues/2805
 [#2806]: https://github.com/DataDog/dd-trace-rb/issues/2806
 [#2810]: https://github.com/DataDog/dd-trace-rb/issues/2810
+[#2815]: https://github.com/DataDog/dd-trace-rb/issues/2815
 [#2822]: https://github.com/DataDog/dd-trace-rb/issues/2822
 [#2824]: https://github.com/DataDog/dd-trace-rb/issues/2824
+[#2826]: https://github.com/DataDog/dd-trace-rb/issues/2826
 [#2829]: https://github.com/DataDog/dd-trace-rb/issues/2829
+[#2836]: https://github.com/DataDog/dd-trace-rb/issues/2836
+[#2840]: https://github.com/DataDog/dd-trace-rb/issues/2840
+[#2854]: https://github.com/DataDog/dd-trace-rb/issues/2854
+[#2855]: https://github.com/DataDog/dd-trace-rb/issues/2855
+[#2856]: https://github.com/DataDog/dd-trace-rb/issues/2856
+[#2858]: https://github.com/DataDog/dd-trace-rb/issues/2858
+[#2860]: https://github.com/DataDog/dd-trace-rb/issues/2860
+[#2864]: https://github.com/DataDog/dd-trace-rb/issues/2864
+[#2866]: https://github.com/DataDog/dd-trace-rb/issues/2866
+[#2869]: https://github.com/DataDog/dd-trace-rb/issues/2869
+[#2873]: https://github.com/DataDog/dd-trace-rb/issues/2873
+[#2874]: https://github.com/DataDog/dd-trace-rb/issues/2874
+[#2875]: https://github.com/DataDog/dd-trace-rb/issues/2875
+[#2883]: https://github.com/DataDog/dd-trace-rb/issues/2883
+[#2890]: https://github.com/DataDog/dd-trace-rb/issues/2890
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot

data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md

@@ -68,19 +68,19 @@ internal types, structures and functions).
 
 Because these private header files are not included in regular Ruby installations, we have two different workarounds:
 
-1. for Ruby versions >= 2.6 we make use use the Ruby private MJIT header
-2. for Ruby versions < 2.6 (legacy Rubies) we make use of the `debase-ruby_core_source` gem
+1. for Ruby versions 2.6 to 3.2 we make use use the Ruby private MJIT header
+2. for Ruby versions < 2.6 and > 3.2 we make use of the `debase-ruby_core_source` gem
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
-There is currently no way for disabling usage of the private MJIT header for Ruby 2.6+.
+There is currently no way for disabling usage of the private MJIT header for Ruby 2.6 to 3.2.
 
 **Important Note**: Our medium/long-term plan is to stop relying on all private Ruby headers, and instead request and
 contribute upstream changes so that they become official public VM APIs.
 
 ### Approach 1: Using the Ruby private MJIT header
 
-Ruby versions >= 2.6 introduced a JIT compiler called MJIT. This compiler does not directly generate machine code;
+Ruby versions 2.6 to 3.2 shipped a JIT compiler called MJIT. This compiler does not directly generate machine code;
 instead it generates C code and uses the system C compiler to turn it into machine code.
 
 The generated C code `#include`s a private header -- which we reference as "the MJIT header" everywhere.
@@ -90,6 +90,8 @@ and of course the intention is that it is only used by the Ruby MJIT compiler.
 This header is placed inside the `include/` directory in a Ruby installation, and is named for that specific Ruby
 version. e.g. `rb_mjit_min_header-2.7.4.h`.
 
+This header was removed in Ruby 3.3.
+
 ### Approach 2: Using the `debase-ruby_core_source` gem
 
 The [`debase-ruby_core_source`](https://github.com/ruby-debug/debase-ruby_core_source) contains almost no code;

data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c

@@ -81,6 +81,7 @@ struct cpu_and_wall_time_worker_state {
 
   bool gc_profiling_enabled;
   bool allocation_counting_enabled;
+  bool no_signals_workaround_enabled;
   bool dynamic_sampling_rate_enabled;
   VALUE self_instance;
   VALUE thread_context_collector_instance;
@@ -148,6 +149,7 @@ static VALUE _native_initialize(
   VALUE gc_profiling_enabled,
   VALUE idle_sampling_helper_instance,
   VALUE allocation_counting_enabled,
+  VALUE no_signals_workaround_enabled,
   VALUE dynamic_sampling_rate_enabled
 );
 static void cpu_and_wall_time_worker_typed_data_mark(void *state_ptr);
@@ -221,7 +223,7 @@ void collectors_cpu_and_wall_time_worker_init(VALUE profiling_module) {
   // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
   rb_define_alloc_func(collectors_cpu_and_wall_time_worker_class, _native_new);
 
-  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 6);
+  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 7);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_sampling_loop", _native_sampling_loop, 1);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_stop", _native_stop, 2);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
@@ -257,6 +259,7 @@ static VALUE _native_new(VALUE klass) {
 
   state->gc_profiling_enabled = false;
   state->allocation_counting_enabled = false;
+  state->no_signals_workaround_enabled = false;
   state->dynamic_sampling_rate_enabled = true;
   state->thread_context_collector_instance = Qnil;
   state->idle_sampling_helper_instance = Qnil;
@@ -283,10 +286,12 @@ static VALUE _native_initialize(
   VALUE gc_profiling_enabled,
   VALUE idle_sampling_helper_instance,
   VALUE allocation_counting_enabled,
+  VALUE no_signals_workaround_enabled,
   VALUE dynamic_sampling_rate_enabled
 ) {
   ENFORCE_BOOLEAN(gc_profiling_enabled);
   ENFORCE_BOOLEAN(allocation_counting_enabled);
+  ENFORCE_BOOLEAN(no_signals_workaround_enabled);
   ENFORCE_BOOLEAN(dynamic_sampling_rate_enabled);
 
   struct cpu_and_wall_time_worker_state *state;
@@ -294,6 +299,7 @@ static VALUE _native_initialize(
 
   state->gc_profiling_enabled = (gc_profiling_enabled == Qtrue);
   state->allocation_counting_enabled = (allocation_counting_enabled == Qtrue);
+  state->no_signals_workaround_enabled = (no_signals_workaround_enabled == Qtrue);
   state->dynamic_sampling_rate_enabled = (dynamic_sampling_rate_enabled == Qtrue);
   state->thread_context_collector_instance = enforce_thread_context_collector_instance(thread_context_collector_instance);
   state->idle_sampling_helper_instance = idle_sampling_helper_instance;
@@ -439,7 +445,7 @@ static void handle_sampling_signal(DDTRACE_UNUSED int _signal, DDTRACE_UNUSED si
   }
 
   // We implicitly assume there can be no concurrent nor nested calls to handle_sampling_signal because
-  // a) we get triggered using SIGPROF, and the docs state second SIGPROF will not interrupt an existing one
+  // a) we get triggered using SIGPROF, and the docs state a second SIGPROF will not interrupt an existing one
   // b) we validate we are in the thread that has the global VM lock; if a different thread gets a signal, it will return early
   //    because it will not have the global VM lock
 
@@ -476,22 +482,34 @@ static void *run_sampling_trigger_loop(void *state_ptr) {
   while (atomic_load(&state->should_run)) {
     state->stats.trigger_sample_attempts++;
 
-    current_gvl_owner owner = gvl_owner();
-    if (owner.valid) {
-      // Note that reading the GVL owner and sending them a signal is a race -- the Ruby VM keeps on executing while
-      // we're doing this, so we may still not signal the correct thread from time to time, but our signal handler
-      // includes a check to see if it got called in the right thread
-      pthread_kill(owner.owner, SIGPROF);
-    } else {
-      // If no thread owns the Global VM Lock, the application is probably idle at the moment. We still want to sample
-      // so we "ask a friend" (the IdleSamplingHelper component) to grab the GVL and simulate getting a SIGPROF.
+    if (state->no_signals_workaround_enabled) {
+      // In the no_signals_workaround_enabled mode, the profiler never sends SIGPROF signals.
+      //
+      // This is a fallback for a few incompatibilities and limitations -- see the code that decides when to enable
+      // `no_signals_workaround_enabled` in `Profiling::Component` for details.
       //
-      // In a previous version of the code, we called `grab_gvl_and_sample` directly BUT this was problematic because
-      // Ruby may concurrently get busy and so the CpuAndWallTimeWorker would be blocked in line to acquire the GVL
-      // for an uncontrolled amount of time. (This can still happen to the IdleSamplingHelper, but the
-      // CpuAndWallTimeWorker will still be free to interrupt the Ruby VM and keep sampling for the entire blocking period).
+      // Thus, we instead pretty please ask Ruby to let us run. This means profiling data can be biased by when the Ruby
+      // scheduler chooses to schedule us.
       state->stats.trigger_simulated_signal_delivery_attempts++;
-      idle_sampling_helper_request_action(state->idle_sampling_helper_instance, grab_gvl_and_sample);
+      grab_gvl_and_sample(); // Note: Can raise exceptions
+    } else {
+      current_gvl_owner owner = gvl_owner();
+      if (owner.valid) {
+        // Note that reading the GVL owner and sending them a signal is a race -- the Ruby VM keeps on executing while
+        // we're doing this, so we may still not signal the correct thread from time to time, but our signal handler
+        // includes a check to see if it got called in the right thread
+        pthread_kill(owner.owner, SIGPROF);
+      } else {
+        // If no thread owns the Global VM Lock, the application is probably idle at the moment. We still want to sample
+        // so we "ask a friend" (the IdleSamplingHelper component) to grab the GVL and simulate getting a SIGPROF.
+        //
+        // In a previous version of the code, we called `grab_gvl_and_sample` directly BUT this was problematic because
+        // Ruby may concurrently get busy and so the CpuAndWallTimeWorker would be blocked in line to acquire the GVL
+        // for an uncontrolled amount of time. (This can still happen to the IdleSamplingHelper, but the
+        // CpuAndWallTimeWorker will still be free to interrupt the Ruby VM and keep sampling for the entire blocking period).
+        state->stats.trigger_simulated_signal_delivery_attempts++;
+        idle_sampling_helper_request_action(state->idle_sampling_helper_instance, grab_gvl_and_sample);
+      }
     }
 
     sleep_for(minimum_time_between_signals);

data/ext/ddtrace_profiling_native_extension/extconf.rb

@@ -81,7 +81,10 @@ end
 
 # Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
 # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
-add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true'
+#
+# @ivoanjo TODO: Ruby 3.3.0-preview1 was causing issues in CI because `have_header('vm_core.h')` below triggers warnings;
+# I've chosen to disable `-Werror` for this Ruby version for now, and we can revisit this on a later 3.3 release.
+add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true' && !RUBY_DESCRIPTION.include?('3.3.0preview1')
 
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
@@ -128,6 +131,12 @@ if RUBY_PLATFORM.include?('linux')
   $defs << '-DHAVE_PTHREAD_GETCPUCLOCKID'
 end
 
+# On older Rubies, we did not need to include the ractor header (this was built into the MJIT header)
+$defs << '-DNO_RACTOR_HEADER_INCLUDE' if RUBY_VERSION < '3.3'
+
+# On older Rubies, some of the Ractor internal APIs were directly accessible
+$defs << '-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY' if RUBY_VERSION < '3.3'
+
 # On older Rubies, there was no struct rb_native_thread. See private_vm_api_acccess.c for details.
 $defs << '-DNO_RB_NATIVE_THREAD' if RUBY_VERSION < '3.2'
 
@@ -217,7 +226,8 @@ if Datadog::Profiling::NativeExtensionHelpers::CAN_USE_MJIT_HEADER
 
   create_makefile EXTENSION_NAME
 else
-  # On older Rubies, we use the debase-ruby_core_source gem to get access to private VM headers.
+  # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
+  # the debase-ruby_core_source gem to get access to private VM headers.
   # This gem ships source code copies of these VM headers for the different Ruby VM versions;
   # see https://github.com/ruby-debug/debase-ruby_core_source for details
 
@@ -228,7 +238,11 @@ else
 
   Debase::RubyCoreSource
     .create_makefile_with_core(
-      proc { have_header('vm_core.h') && have_header('iseq.h') },
+      proc do
+        have_header('vm_core.h') &&
+        have_header('iseq.h') &&
+        (RUBY_VERSION < '3.3' || have_header('ractor_core.h'))
+      end,
       EXTENSION_NAME,
     )
 end

data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb

@@ -12,8 +12,8 @@ module Datadog
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = 'DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION'
 
-      # Older Rubies don't have the MJIT header, used by the JIT compiler, so we need to use a different approach
-      CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
+      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
+      CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?('2.6', '2.7', '3.0.', '3.1.', '3.2.')
 
       LIBDATADOG_VERSION = '~> 2.0.0.1.0'
 

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c

@@ -12,7 +12,8 @@
   // Pick up internal structures from the private Ruby MJIT header file
   #include RUBY_MJIT_HEADER
 #else
-  // On older Rubies, use a copy of the VM internal headers shipped in the debase-ruby_core_source gem
+  // The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
+  // the debase-ruby_core_source gem to get access to private VM headers.
 
   // We can't do anything about warnings in VM headers, so we just use this technique to suppress them.
   // See https://nelkinda.com/blog/suppress-warnings-in-gcc-and-clang/#d11e364 for details.
@@ -23,7 +24,20 @@
   #pragma GCC diagnostic ignored "-Wexpansion-to-defined"
     #include <vm_core.h>
   #pragma GCC diagnostic pop
-  #include <iseq.h>
+
+  #pragma GCC diagnostic push
+  #pragma GCC diagnostic ignored "-Wunused-parameter"
+    #include <iseq.h>
+  #pragma GCC diagnostic pop
+
+  #include <ruby.h>
+
+  #ifndef NO_RACTOR_HEADER_INCLUDE
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wunused-parameter"
+      #include <ractor_core.h>
+    #pragma GCC diagnostic pop
+  #endif
 #endif
 
 #define PRIVATE_VM_API_ACCESS_SKIP_RUBY_INCLUDES
@@ -712,8 +726,28 @@ check_method_entry(VALUE obj, int can_be_svar)
 #ifndef NO_RACTORS
   // This API and definition are exported as a public symbol by the VM BUT the function header is not defined in any public header, so we
   // repeat it here to be able to use in our code.
-  bool rb_ractor_main_p_(void);
-  extern struct rb_ractor_struct *ruby_single_main_ractor;
+  #ifndef USE_RACTOR_INTERNAL_APIS_DIRECTLY
+    // Disable fast path for detecting multiple Ractors. Unfortunately this symbol is no longer visible on modern Ruby
+    // versions, so we need to do a bit more work.
+    struct rb_ractor_struct *ruby_single_main_ractor = NULL;
+
+    // Taken from upstream ractor.c at commit a1b01e7701f9fc370f8dff777aad6d39a2c5a3e3 (May 2023, Ruby 3.3.0-preview1)
+    // to allow us to ensure that we're always operating on the main ractor (if Ruby has ractors)
+    // Modifications:
+    // * None
+    bool rb_ractor_main_p_(void)
+    {
+        VM_ASSERT(rb_multi_ractor_p());
+        rb_execution_context_t *ec = GET_EC();
+        return rb_ec_ractor_ptr(ec) == rb_ec_vm_ptr(ec)->ractor.main_ractor;
+    }
+  #else
+    // Directly access Ruby internal fast path for detecting multiple Ractors.
+    extern struct rb_ractor_struct *ruby_single_main_ractor;
+
+    // Ruby 3.0 to 3.2 directly expose this symbol, we just need to tell the compiler it exists.
+    bool rb_ractor_main_p_(void);
+  #endif
 
   // Taken from upstream ractor_core.h at commit d9cf0388599a3234b9f3c06ddd006cd59a58ab8b (November 2022, Ruby 3.2 trunk)
   // to allow us to ensure that we're always operating on the main ractor (if Ruby has ractors)