datadog 2.8.0 → 2.39.0

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -69,20 +69,32 @@ $stderr.puts(
 # NOTE: we MUST NOT require 'mkmf' before we check the #skip_building_extension? because the require triggers checks
 # that may fail on an environment not properly setup for building Ruby extensions.
 require "mkmf"
+Datadog::LibdatadogExtconfHelpers.dump_mkmf_log_on_failure!
 
 Logging.message("[datadog] Using compiler:\n")
 xsystem("#{CONFIG["CC"]} -v")
 Logging.message("[datadog] End of compiler information\n")
 
+# We must *never* `append_cflags "-Wall"` or `append_cflags "-Wextra"`, because those add the flag at the end,
+# which then overrides earlier -Wno-* flags from RbConfig::CONFIG["warnflags"], and causes errors if -Werror is added.
+# There is no need to add them anyway, Ruby since 1.9 has RbConfig::CONFIG["warnflags"] starting with "-Wall -Wextra ".
+
 # Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
 # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
 append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
 
+# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
+# (a header-dedup regression), which `-Werror` turns into fatal
+# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
+# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
+append_cflags "-Wno-error=typedef-redefinition" if ENV["DATADOG_GEM_CI"] == "true"
+
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
 # * by msgpack, another datadog gem dependency
 #   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
-append_cflags "-std=gnu99"
+# @ivoanjo: We could probably start using C11/gnu11 for non macOS-too but it's somewhat hard to validate so I chickened out for now
+append_cflags RUBY_PLATFORM.include?("darwin") ? "-std=gnu11" : "-std=gnu99"
 
 # Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 append_cflags "-Wno-unused-function"
@@ -105,10 +117,6 @@ append_cflags "-fvisibility=hidden"
 # Avoid legacy C definitions
 append_cflags "-Wold-style-definition"
 
-# Enable all other compiler warnings
-append_cflags "-Wall"
-append_cflags "-Wextra"
-
 if ENV["DDTRACE_DEBUG"] == "true"
   $defs << "-DDD_DEBUG"
   CONFIG["optflags"] = "-O0"
@@ -124,15 +132,40 @@ if RUBY_PLATFORM.include?("linux")
   # but it's slower to build
   # so instead we just assume that we have the function we need on Linux, and nowhere else
   $defs << "-DHAVE_PTHREAD_GETCPUCLOCKID"
-
-  # Not available on macOS
-  $defs << "-DHAVE_CLOCK_MONOTONIC_COARSE"
+elsif RUBY_PLATFORM.include?("darwin")
+  # On macOS, we use Mach thread APIs to get per-thread CPU time
+  $defs << "-DHAVE_MACH_THREAD_INFO"
 end
 
 have_func "malloc_stats"
 
-# On Ruby 2.5 and 3.3, this symbol was not visible. It is on 2.6 to 3.2, as well as 3.4+
-$defs << "-DNO_RB_OBJ_INFO" if RUBY_VERSION.start_with?("2.5", "3.3")
+# Used to get native filenames (dladdr1 is preferred, so we only check for the other if not available)
+# Note it's possible none are available
+if have_header("dlfcn.h")
+  (have_struct_member("struct link_map", "l_name", "link.h") && have_func("dladdr1")) ||
+    have_func("dladdr")
+end
+
+# On older Rubies, there was no primitive mutex and condition variable implemented in `thread_sync.rb` (internal)
+$defs << "-DNO_PRIMITIVE_MUTEX_AND_CONDITION_VARIABLE" if RUBY_VERSION < "4"
+
+# On Ruby 4, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
+$defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
+
+# On Ruby 4, we need to defer calling rb_obj_id during heap allocation recording
+# because it's not safe to mutate objects during the newobj tracepoint
+# (see https://bugs.ruby-lang.org/issues/21710)
+$defs << "-DUSE_DEFERRED_HEAP_ALLOCATION_RECORDING" unless RUBY_VERSION < "4"
+
+# On Ruby 4.0, we've seen crashes when computing the memsize of a class/module/iclass:
+# rb_obj_memsize_of walks the per-namespace class extensions (classext_memsize), which seem to sometimes be in an inconsistent state
+# (see https://github.com/DataDog/dd-trace-rb/issues/5936)
+$defs << "-DNO_SAFE_CLASS_MEMSIZE" unless RUBY_VERSION < "4"
+
+# This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
+# It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
+# actually required for correct behavior of the profiler.
+$defs << "-DNO_RB_OBJ_INFO" if RUBY_VERSION.start_with?("2.5", "3.3", "4.")
 
 # On older Rubies, rb_postponed_job_preregister/rb_postponed_job_trigger did not exist
 $defs << "-DNO_POSTPONED_TRIGGER" if RUBY_VERSION < "3.3"
@@ -149,8 +182,8 @@ $defs << "-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY" if RUBY_VERSION < "3.3"
 # On older Rubies, there was no GVL instrumentation API and APIs created to support it
 $defs << "-DNO_GVL_INSTRUMENTATION" if RUBY_VERSION < "3.2"
 
-# Supporting GVL instrumentation on 3.2 needs some workarounds
-$defs << "-DUSE_GVL_PROFILING_3_2_WORKAROUNDS" if RUBY_VERSION.start_with?("3.2")
+# rb_internal_thread_specific_*()
+$defs << "-DHAVE_RUBY_THREAD_STORAGE_API" if RUBY_VERSION >= "3.3"
 
 # On older Rubies, there was no struct rb_native_thread. See private_vm_api_acccess.c for details.
 $defs << "-DNO_RB_NATIVE_THREAD" if RUBY_VERSION < "3.2"
@@ -164,6 +197,10 @@ $defs << "-DNO_INT_FIRST_LINENO" if RUBY_VERSION < "3.2"
 # On older Rubies, "pop" was not a primitive operation
 $defs << "-DNO_PRIMITIVE_POP" if RUBY_VERSION < "3.2"
 
+# We could support this for older Rubies, but since this only gets used by the OTEL context extraction, and that
+# use-case is only for 3.1+, we didn't bother supporting it farther back yet.
+$defs << "-DNO_CURRENT_FIBER_FOR" if RUBY_VERSION < "3.1"
+
 # On older Rubies, there was no tid member in the internal thread structure
 $defs << "-DNO_THREAD_TID" if RUBY_VERSION < "3.1"
 
@@ -186,37 +223,18 @@ $defs << "-DNO_GVL_OWNER" if RUBY_VERSION < "2.6"
 $defs << "-DNO_THREAD_INVOKE_ARG" if RUBY_VERSION < "2.6"
 
 # If we got here, libdatadog is available and loaded
-ENV["PKG_CONFIG_PATH"] = "#{ENV["PKG_CONFIG_PATH"]}:#{Libdatadog.pkgconfig_folder}"
-Logging.message("[datadog] PKG_CONFIG_PATH set to #{ENV["PKG_CONFIG_PATH"].inspect}\n")
 $stderr.puts("Using libdatadog #{Libdatadog::VERSION} from #{Libdatadog.pkgconfig_folder}")
 
-unless pkg_config("datadog_profiling_with_rpath")
-  Logging.message("[datadog] Ruby detected the pkg-config command is #{$PKGCONFIG.inspect}\n")
-
-  skip_building_extension!(
-    if Datadog::LibdatadogExtconfHelpers.pkg_config_missing?
-      Datadog::Profiling::NativeExtensionHelpers::Supported::PKG_CONFIG_IS_MISSING
-    else
-      # Less specific error message
-      Datadog::Profiling::NativeExtensionHelpers::Supported::FAILED_TO_CONFIGURE_LIBDATADOG
-    end
-  )
+unless Datadog::LibdatadogExtconfHelpers.configure_libdatadog(extconf_folder: __dir__)
+  skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::FAILED_TO_CONFIGURE_LIBDATADOG)
 end
 
+Datadog::LibdatadogExtconfHelpers.add_libdatadog_version_define
+
 unless have_type("atomic_int", ["stdatomic.h"])
   skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::COMPILER_ATOMIC_MISSING)
 end
 
-# See comments on the helper methods being used for why we need to additionally set this.
-# The extremely excessive escaping around ORIGIN below seems to be correct and was determined after a lot of
-# experimentation. We need to get these special characters across a lot of tools untouched...
-extra_relative_rpaths = [
-  Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_native_lib_folder(current_folder: __dir__),
-  *Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_ruby_extensions_folders,
-]
-extra_relative_rpaths.each { |folder| $LDFLAGS += " -Wl,-rpath,$$$\\\\{ORIGIN\\}/#{folder.to_str}" }
-Logging.message("[datadog] After pkg-config $LDFLAGS were set to: #{$LDFLAGS.inspect}\n")
-
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
 # the wrong library is never loaded.

data/ext/datadog_profiling_native_extension/gvl_profiling_helper.c

@@ -1,50 +1,13 @@
 #include <ruby.h>
 #include <ruby/thread.h>
-#include "gvl_profiling_helper.h"
-
-#if !defined(NO_GVL_INSTRUMENTATION) && !defined(USE_GVL_PROFILING_3_2_WORKAROUNDS) // Ruby 3.3+
-  rb_internal_thread_specific_key_t gvl_waiting_tls_key;
-
-  void gvl_profiling_init(void) {
-    gvl_waiting_tls_key = rb_internal_thread_specific_key_create();
-  }
-
-#endif
-
-#ifdef USE_GVL_PROFILING_3_2_WORKAROUNDS // Ruby 3.2
-  __thread gvl_profiling_thread gvl_waiting_tls;
-  static bool gvl_profiling_state_thread_tracking_workaround_installed = false;
 
-  static void on_thread_start(
-    DDTRACE_UNUSED rb_event_flag_t _unused1,
-    DDTRACE_UNUSED const rb_internal_thread_event_data_t *_unused2,
-    DDTRACE_UNUSED void *_unused3
-  ) {
-    gvl_waiting_tls = (gvl_profiling_thread) {.thread = NULL};
-  }
-
-  // Hack: We're using the gvl_waiting_tls native thread-local to store per-thread information. Unfortunately, Ruby puts a big hole
-  // in our plan because it reuses native threads -- specifically, in Ruby 3.2, native threads are still 1:1 to Ruby
-  // threads (M:N wasn't a thing yet) BUT once a Ruby thread dies, the VM will keep the native thread around for a
-  // bit, and if another Ruby thread starts right after, Ruby will reuse the native thread, rather than create a new one.
-  //
-  // This will mean that the new Ruby thread will still have the same native thread-local data that we set on the
-  // old thread. For the purposes of our tracking, where we're keeping a pointer to the current thread object in
-  // thread-local storage **this is disastrous** since it means we'll be pointing at the wrong thread (and its
-  // memory may have been freed or reused since!)
-  //
-  // To work around this issue, once GVL profiling is enabled, we install an event hook on thread start
-  // events that clears the thread-local data. This guarantees that there will be no stale data -- any existing
-  // data will be cleared at thread start.
-  //
-  // Note that once installed, this event hook becomes permanent -- stopping the profiler does not stop this event
-  // hook, unlike all others. This is because we can't afford to miss any thread start events while the
-  // profiler is stopped (e.g. during reconfiguration) as that would mean stale data once the profiler starts again.
-  void gvl_profiling_state_thread_tracking_workaround(void) {
-    if (gvl_profiling_state_thread_tracking_workaround_installed) return;
+#include "datadog_ruby_common.h"
+#include "gvl_profiling_helper.h"
 
-    rb_internal_thread_add_event_hook(on_thread_start, RUBY_INTERNAL_THREAD_EVENT_STARTED, NULL);
+#ifdef HAVE_RUBY_THREAD_STORAGE_API
+  rb_internal_thread_specific_key_t per_thread_context_key;
 
-    gvl_profiling_state_thread_tracking_workaround_installed = true;
+  void per_thread_context_tls_init(void) {
+    per_thread_context_key = rb_internal_thread_specific_key_create();
   }
 #endif

data/ext/datadog_profiling_native_extension/gvl_profiling_helper.h

@@ -6,70 +6,30 @@
 
 #include "extconf.h"
 
-#if !defined(NO_GVL_INSTRUMENTATION) && !defined(USE_GVL_PROFILING_3_2_WORKAROUNDS) // Ruby 3.3+
-  #include <ruby.h>
-  #include <ruby/thread.h>
-  #include "datadog_ruby_common.h"
-
-  typedef struct { VALUE thread; } gvl_profiling_thread;
-  extern rb_internal_thread_specific_key_t gvl_waiting_tls_key;
+#include <ruby.h>
 
-  void gvl_profiling_init(void);
+// Opaque to this header; the full definition lives in collectors_thread_context.c.
+typedef struct per_thread_context per_thread_context;
 
-  static inline gvl_profiling_thread thread_from_thread_object(VALUE thread) {
-    return (gvl_profiling_thread) {.thread = thread};
-  }
+#ifdef HAVE_RUBY_THREAD_STORAGE_API
+  #include <ruby/thread.h>
 
-  static inline gvl_profiling_thread thread_from_event(const rb_internal_thread_event_data_t *event_data) {
-    return thread_from_thread_object(event_data->thread);
-  }
+  extern rb_internal_thread_specific_key_t per_thread_context_key;
 
-  static inline intptr_t gvl_profiling_state_get(gvl_profiling_thread thread) {
-    return (intptr_t) rb_internal_thread_specific_get(thread.thread, gvl_waiting_tls_key);
-  }
+  void per_thread_context_tls_init(void);
 
-  static inline void gvl_profiling_state_set(gvl_profiling_thread thread, intptr_t value) {
-    rb_internal_thread_specific_set(thread.thread, gvl_waiting_tls_key, (void *) value);
+  static inline per_thread_context* get_per_thread_context(VALUE thread) {
+    return rb_internal_thread_specific_get(thread, per_thread_context_key);
   }
-#endif
-
-#ifdef USE_GVL_PROFILING_3_2_WORKAROUNDS // Ruby 3.2
-  typedef struct { void *thread; } gvl_profiling_thread;
-  extern __thread gvl_profiling_thread gvl_waiting_tls;
 
-  static inline void gvl_profiling_init(void) { }
-
-  // This header gets included in private_vm_access.c which can't include datadog_ruby_common.h so we replicate this
-  // helper here
-  #ifdef __GNUC__
-    #define DDTRACE_UNUSED  __attribute__((unused))
-  #else
-    #define DDTRACE_UNUSED
-  #endif
-
-  // NOTE: This is a hack that relies on the knowledge that on Ruby 3.2 the
-  // RUBY_INTERNAL_THREAD_EVENT_READY and RUBY_INTERNAL_THREAD_EVENT_RESUMED events always get called on the thread they
-  // are about. Thus, we can use our thread local storage hack to get this data, even though the event doesn't include it.
-  static inline gvl_profiling_thread thread_from_event(DDTRACE_UNUSED const void *event_data) {
-    return gvl_waiting_tls;
+  static inline void set_per_thread_context(VALUE thread, per_thread_context* value) {
+    rb_internal_thread_specific_set(thread, per_thread_context_key, value);
   }
-
-  void gvl_profiling_state_thread_tracking_workaround(void);
-  gvl_profiling_thread gvl_profiling_state_maybe_initialize(void);
+#else
+  static inline void per_thread_context_tls_init(void) { }
 
   // Implementing these on Ruby 3.2 requires access to private VM things, so the following methods are
   // implemented in `private_vm_api_access.c`
-  gvl_profiling_thread thread_from_thread_object(VALUE thread);
-  intptr_t gvl_profiling_state_get(gvl_profiling_thread thread);
-  void gvl_profiling_state_set(gvl_profiling_thread thread, intptr_t value);
-#endif
-
-#ifndef NO_GVL_INSTRUMENTATION // For all Rubies supporting GVL profiling (3.2+)
-  static inline intptr_t gvl_profiling_state_thread_object_get(VALUE thread) {
-    return gvl_profiling_state_get(thread_from_thread_object(thread));
-  }
-
-  static inline void gvl_profiling_state_thread_object_set(VALUE thread, intptr_t value) {
-    gvl_profiling_state_set(thread_from_thread_object(thread), value);
-  }
+  per_thread_context* get_per_thread_context(VALUE thread);
+  void set_per_thread_context(VALUE thread, per_thread_context* value);
 #endif