datadog 2.30.0 → 2.45.0

data/ext/datadog_profiling_native_extension/collectors_thread_context.h

@@ -2,32 +2,48 @@
 
 #include <ruby.h>
 #include <stdbool.h>
+#include <stdint.h>
 
 #include "gvl_profiling_helper.h"
 
-void thread_context_collector_sample(
+// Returns true if `thread_context_collector_resolve_otel_span_key_may_lose_gvl` needs to be called (which the caller
+// must only do once the sample is over)
+__attribute__((warn_unused_result)) bool thread_context_collector_sample(
   VALUE self_instance,
-  long current_monotonic_wall_time_ns,
-  VALUE profiler_overhead_stack_thread
+  long current_monotonic_wall_time_ns
 );
-__attribute__((warn_unused_result)) bool thread_context_collector_prepare_sample_inside_signal_handler(VALUE self_instance);
-__attribute__((warn_unused_result)) bool thread_context_collector_sample_allocation(VALUE self_instance, unsigned int sample_weight, VALUE new_object);
-void thread_context_collector_after_allocation(VALUE self_instance);
+__attribute__((warn_unused_result)) bool thread_context_collector_prepare_sample_inside_signal_handler(void);
+void thread_context_collector_prepare_sample_on_gc_finish(void);
+__attribute__((warn_unused_result)) bool thread_context_collector_sample_allocation(VALUE self_instance, per_thread_context *thread_context, unsigned int sample_weight, VALUE new_object);
+void thread_context_collector_commit_heap_recordings_may_lose_gvl(VALUE self_instance);
 void thread_context_collector_sample_skipped_allocation_samples(VALUE self_instance, unsigned int skipped_samples);
 VALUE thread_context_collector_sample_after_gc(VALUE self_instance);
+VALUE thread_context_collector_heap_update_may_lose_gvl(VALUE self_instance);
+void thread_context_collector_resolve_otel_span_key_may_lose_gvl(VALUE self_instance);
 void thread_context_collector_on_gc_start(VALUE self_instance);
 __attribute__((warn_unused_result)) bool thread_context_collector_on_gc_finish(VALUE self_instance);
 VALUE enforce_thread_context_collector_instance(VALUE object);
-
+void thread_context_collector_stats(VALUE self_instance, VALUE stats_hash);
+void thread_context_collector_stats_reset_not_thread_safe(VALUE self_instance);
+VALUE thread_context_prepare_serialize(VALUE self_instance);
+void thread_context_collector_reset_all_per_thread_contexts(VALUE self_instance);
+void thread_context_collector_profiler_internal_thread_started(void);
+VALUE thread_context_collector_profiler_internal_thread_done(VALUE self_instance);
 
 #ifndef NO_GVL_INSTRUMENTATION
   typedef enum {
     ON_GVL_RUNNING_UNKNOWN, // Thread is not known, it may not even be from the current Ractor
     ON_GVL_RUNNING_DONT_SAMPLE, // Thread is known, but "Waiting for GVL" period was too small to be sampled
     ON_GVL_RUNNING_SAMPLE, // Thread is known, and "Waiting for GVL" period should be sampled
+  } on_gvl_running_enum;
+
+  typedef struct {
+    on_gvl_running_enum action;
+    long waiting_for_gvl_duration_ns;
   } on_gvl_running_result;
 
-  void thread_context_collector_on_gvl_waiting(gvl_profiling_thread thread);
-  __attribute__((warn_unused_result)) on_gvl_running_result thread_context_collector_on_gvl_running(gvl_profiling_thread thread);
+  void thread_context_collector_on_gvl_waiting(per_thread_context *thread_context);
+  __attribute__((warn_unused_result)) on_gvl_running_result thread_context_collector_on_gvl_running(VALUE thread, per_thread_context *thread_context, uint32_t waiting_for_gvl_threshold_ns);
   VALUE thread_context_collector_sample_after_gvl_running(VALUE self_instance, VALUE current_thread, long current_monotonic_wall_time_ns);
+  void thread_context_collector_on_gvl_released(per_thread_context *thread_context);
 #endif

data/ext/datadog_profiling_native_extension/crashtracking_runtime_stacks.c

@@ -159,7 +159,7 @@ static void ruby_runtime_stack_callback(
     frame_info *info = &runtime_stack_buffer[i];
 
     if (info->is_ruby_frame) {
-      const void *iseq = (const void *)info->as.ruby_frame.iseq;
+      const rb_iseq_t *iseq = info->as.ruby_frame.iseq;
       ddog_CharSlice function_slice = DDOG_CHARSLICE_C("<unknown>");
       ddog_CharSlice file_slice = DDOG_CHARSLICE_C("<unknown>");
 
@@ -181,9 +181,10 @@ static void ruby_runtime_stack_callback(
       ddog_CharSlice function_slice = DDOG_CHARSLICE_C("<C method>");
       ddog_CharSlice file_slice = DDOG_CHARSLICE_C("<C extension>");
 
-      if (info->as.native_frame.method_id) {
-        const char *method_name = rb_id2name(info->as.native_frame.method_id);
-        if (is_pointer_readable(method_name, 256)) {
+      const rb_callable_method_entry_t *cme = info->cme;
+      if (cme && is_pointer_readable(cme, sizeof_rb_callable_method_entry_t())) {
+        const char *method_name = ddtrace_cme_original_method_name(cme);
+        if (method_name && is_pointer_readable(method_name, 256)) {
           size_t method_name_len = strnlen(method_name, 256);
           if (method_name_len > 0 && method_name_len < 256) {
             function_slice = char_slice_from_cstr(method_name);

data/ext/datadog_profiling_native_extension/datadog_ruby_common.c

@@ -6,19 +6,27 @@
 static ID telemetry_message_id;
 
 void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char *function_name) {
-  rb_exc_raise(
-    rb_exc_new_str(
-      rb_eTypeError,
-      rb_sprintf("wrong argument %"PRIsVALUE" for '%s' (expected a %s) at %s:%d:in `%s'",
-        rb_inspect(value),
-        value_name,
-        type_name,
-        file,
-        line,
-        function_name
-      )
+  // All of the below come from `ENFORCE_TYPE` and are constant for their call-site
+  VALUE simple_message = rb_sprintf("wrong argument for '%s' (expected a %s) at %s:%d:in `%s'",
+    value_name,
+    type_name,
+    file,
+    line,
+    function_name
+  );
+  VALUE exception = rb_exc_new_str(
+    rb_eTypeError,
+    rb_sprintf("wrong argument %"PRIsVALUE" for '%s' (expected a %s) at %s:%d:in `%s'",
+      rb_inspect(value),
+      value_name,
+      type_name,
+      file,
+      line,
+      function_name
     )
   );
+  rb_ivar_set(exception, telemetry_message_id, simple_message);
+  rb_exc_raise(exception);
 }
 
 // Raises an exception with separate telemetry-safe and detailed messages.
@@ -29,16 +37,15 @@ void private_raise_exception(VALUE exception, const char *static_message) {
   rb_exc_raise(exception);
 }
 
-// Helper for raising pre-formatted exceptions
-void private_raise_error_formatted(VALUE exception_class, const char *detailed_message, const char *static_message) {
-  VALUE exception = rb_exc_new_cstr(exception_class, detailed_message);
-  private_raise_exception(exception, static_message);
-}
-
 // Use `raise_error` the macro instead, as it provides additional argument checks.
 void private_raise_error(VALUE exception_class, const char *fmt, ...) {
-  FORMAT_VA_ERROR_MESSAGE(detailed_message, fmt);
-  private_raise_error_formatted(exception_class, detailed_message, fmt);
+  va_list args;
+  va_start(args, fmt);
+  VALUE detailed_message = rb_vsprintf(fmt, args);
+  va_end(args);
+
+  VALUE exception = rb_exc_new_str(exception_class, detailed_message);
+  private_raise_exception(exception, fmt);
 }
 
 VALUE datadog_gem_version(void) {
@@ -122,6 +129,24 @@ size_t read_ddogerr_string_and_drop(ddog_Error *error, char *string, size_t capa
   return error_msg_size;
 }
 
+static void verify_libdatadog_version(void) {
+  rb_eval_string(
+    "require 'libdatadog';"
+    "expected = '" EXPECTED_LIBDATADOG_VERSION "';"
+    "if expected != Libdatadog::VERSION;"
+      "raise(LoadError, <<MSG\n"
+        "The `datadog` gem needs to be reinstalled whenever the `libdatadog` gem version is changed. "
+        "The currently-installed version of `datadog` was built to work with `libdatadog` gem version #{expected} "
+        "but the currently-loaded version of `libdatadog` is #{Libdatadog::VERSION}. "
+        "To fix this, reinstall the `datadog` gem (e.g. `bundle exec gem pristine datadog`) "
+        "or contact Datadog support for help at <https://docs.datadoghq.com/help/>.\n"
+        "MSG\n"
+      ");"
+    "end"
+  );
+}
+
 void datadog_ruby_common_init(void) {
   telemetry_message_id = rb_intern("@telemetry_message");
+  verify_libdatadog_version();
 }

data/ext/datadog_profiling_native_extension/datadog_ruby_common.h

@@ -5,6 +5,13 @@
 #include <ruby.h>
 #include <datadog/common.h>
 
+// All recommended backward compatible macros for keyword argument functions can be found here:
+// https://github.com/ruby/ruby/blob/master/doc/extension.rdoc#defining-backward-compatible-macros-for-keyword-argument-functions
+#ifndef RB_PASS_KEYWORDS
+/* Only define macros on Ruby <2.7 */
+#define rb_funcallv_kw(o, m, c, v, kw) rb_funcallv(o, m, c, v)
+#endif
+
 // Must be called once during initialization
 void datadog_ruby_common_init(void);
 
@@ -47,11 +54,6 @@ NORETURN(
   __attribute__ ((format (printf, 2, 3)));
 );
 
-// Internal helper for raising pre-formatted exceptions
-NORETURN(
-  void private_raise_error_formatted(VALUE exception_class, const char *detailed_message, const char *static_message)
-);
-
 // Raises an exception with separate telemetry-safe and detailed messages.
 // NOTE: Raising an exception always invokes Ruby code so it requires the GVL and is not compatible with "debug_enter_unsafe_context".
 // @see debug_enter_unsafe_context
@@ -61,13 +63,6 @@ NORETURN(
 
 #define MAX_RAISE_MESSAGE_SIZE 256
 
-#define FORMAT_VA_ERROR_MESSAGE(buf, fmt) \
-  char buf[MAX_RAISE_MESSAGE_SIZE]; \
-  va_list buf##_args; \
-  va_start(buf##_args, fmt); \
-  vsnprintf(buf, MAX_RAISE_MESSAGE_SIZE, fmt, buf##_args); \
-  va_end(buf##_args);
-
 // Helper to retrieve Datadog::VERSION::STRING
 VALUE datadog_gem_version(void);
 

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -69,23 +69,34 @@ $stderr.puts(
 # NOTE: we MUST NOT require 'mkmf' before we check the #skip_building_extension? because the require triggers checks
 # that may fail on an environment not properly setup for building Ruby extensions.
 require "mkmf"
+Datadog::LibdatadogExtconfHelpers.dump_mkmf_log_on_failure!
 
 Logging.message("[datadog] Using compiler:\n")
 xsystem("#{CONFIG["CC"]} -v")
 Logging.message("[datadog] End of compiler information\n")
 
+# We must *never* `append_cflags "-Wall"` or `append_cflags "-Wextra"`, because those add the flag at the end,
+# which then overrides earlier -Wno-* flags from RbConfig::CONFIG["warnflags"], and causes errors if -Werror is added.
+# There is no need to add them anyway, Ruby since 1.9 has RbConfig::CONFIG["warnflags"] starting with "-Wall -Wextra ".
+
 # Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
 # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
 append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
 
+# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
+# (a header-dedup regression), which `-Werror` turns into fatal
+# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
+# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
+append_cflags "-Wno-error=typedef-redefinition" if ENV["DATADOG_GEM_CI"] == "true"
+
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
 # * by msgpack, another datadog gem dependency
 #   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
 # @ivoanjo: We could probably start using C11/gnu11 for non macOS-too but it's somewhat hard to validate so I chickened out for now
-append_cflags RUBY_PLATFORM.include?('darwin') ? '-std=gnu11' : '-std=gnu99'
+append_cflags RUBY_PLATFORM.include?("darwin") ? "-std=gnu11" : "-std=gnu99"
 
-# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
+# Gets really noisy when we include the private VM headers, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 append_cflags "-Wno-unused-function"
 
 # Allow defining variables at any point in a function
@@ -106,10 +117,6 @@ append_cflags "-fvisibility=hidden"
 # Avoid legacy C definitions
 append_cflags "-Wold-style-definition"
 
-# Enable all other compiler warnings
-append_cflags "-Wall"
-append_cflags "-Wextra"
-
 if ENV["DDTRACE_DEBUG"] == "true"
   $defs << "-DDD_DEBUG"
   CONFIG["optflags"] = "-O0"
@@ -125,31 +132,18 @@ if RUBY_PLATFORM.include?("linux")
   # but it's slower to build
   # so instead we just assume that we have the function we need on Linux, and nowhere else
   $defs << "-DHAVE_PTHREAD_GETCPUCLOCKID"
-
-  # Not available on macOS
-  $defs << "-DHAVE_CLOCK_MONOTONIC_COARSE"
+elsif RUBY_PLATFORM.include?("darwin")
+  # On macOS, we use Mach thread APIs to get per-thread CPU time
+  $defs << "-DHAVE_MACH_THREAD_INFO"
 end
 
 have_func "malloc_stats"
 
-# Used to get native filenames (dladdr1 is preferred, so we only check for the other if not available)
-# Note it's possible none are available
-if have_header("dlfcn.h")
-  (have_struct_member("struct link_map", "l_name", "link.h") && have_func("dladdr1")) ||
-    have_func("dladdr")
-end
+$defs << "-DHAVE_ZJIT_FRAME" if RUBY_VERSION >= "4.1"
 
 # On older Rubies, there was no primitive mutex and condition variable implemented in `thread_sync.rb` (internal)
 $defs << "-DNO_PRIMITIVE_MUTEX_AND_CONDITION_VARIABLE" if RUBY_VERSION < "4"
 
-# On Ruby 4, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
-$defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
-
-# On Ruby 4, we need to defer calling rb_obj_id during heap allocation recording
-# because it's not safe to mutate objects during the newobj tracepoint
-# (see https://bugs.ruby-lang.org/issues/21710)
-$defs << "-DUSE_DEFERRED_HEAP_ALLOCATION_RECORDING" unless RUBY_VERSION < "4"
-
 # This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
 # It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
 # actually required for correct behavior of the profiler.
@@ -161,8 +155,8 @@ $defs << "-DNO_POSTPONED_TRIGGER" if RUBY_VERSION < "3.3"
 # On older Rubies, M:N threads were not available
 $defs << "-DNO_MN_THREADS_AVAILABLE" if RUBY_VERSION < "3.3"
 
-# On older Rubies, we did not need to include the ractor header (this was built into the MJIT header)
-$defs << "-DNO_RACTOR_HEADER_INCLUDE" if RUBY_VERSION < "3.3"
+# On Rubies before 3.0, there were no Ractors, so there's no need to include the ractor header at all
+$defs << "-DNO_RACTOR_HEADER_INCLUDE" if RUBY_VERSION < "3"
 
 # On older Rubies, some of the Ractor internal APIs were directly accessible
 $defs << "-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY" if RUBY_VERSION < "3.3"
@@ -170,8 +164,17 @@ $defs << "-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY" if RUBY_VERSION < "3.3"
 # On older Rubies, there was no GVL instrumentation API and APIs created to support it
 $defs << "-DNO_GVL_INSTRUMENTATION" if RUBY_VERSION < "3.2"
 
-# Supporting GVL instrumentation on 3.2 needs some workarounds
-$defs << "-DUSE_GVL_PROFILING_3_2_WORKAROUNDS" if RUBY_VERSION.start_with?("3.2")
+# On older Rubies, rb_class_attached_object is not available
+$defs << "-DNO_CLASS_ATTACHED_OBJECT" if RUBY_VERSION < "3.2"
+
+# On older Rubies, rb_mod_name allocates (rb_str_dup); use rb_attr_get(__classpath__) instead
+$defs << "-DNO_ALLOC_FREE_MOD_NAME" if RUBY_VERSION < "2.7"
+
+# rb_internal_thread_specific_*()
+$defs << "-DHAVE_RUBY_THREAD_STORAGE_API" if RUBY_VERSION >= "3.3"
+
+# RCLASS_EXT(mod)->permanent_classpath
+$defs << "-DHAVE_PERMANENT_CLASSPATH" if RUBY_VERSION >= "3.3"
 
 # On older Rubies, there was no struct rb_native_thread. See private_vm_api_acccess.c for details.
 $defs << "-DNO_RB_NATIVE_THREAD" if RUBY_VERSION < "3.2"
@@ -195,6 +198,9 @@ $defs << "-DNO_THREAD_TID" if RUBY_VERSION < "3.1"
 # On older Rubies, there was no jit_return member on the rb_control_frame_t struct
 $defs << "-DNO_JIT_RETURN" if RUBY_VERSION < "3.1"
 
+# internal/class.h is available since 3.0
+$defs << "-DNO_INTERNAL_CLASS_HEADER_INCLUDE" if RUBY_VERSION < "3"
+
 # On older Rubies, there are no Ractors
 $defs << "-DNO_RACTORS" if RUBY_VERSION < "3"
 
@@ -217,6 +223,8 @@ unless Datadog::LibdatadogExtconfHelpers.configure_libdatadog(extconf_folder: __
   skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::FAILED_TO_CONFIGURE_LIBDATADOG)
 end
 
+Datadog::LibdatadogExtconfHelpers.add_libdatadog_version_define
+
 unless have_type("atomic_int", ["stdatomic.h"])
   skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::COMPILER_ATOMIC_MISSING)
 end
@@ -227,85 +235,59 @@ end
 # When requiring, we need to use the exact same string, including the version and the platform.
 EXTENSION_NAME = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}".freeze
 
-if Datadog::Profiling::NativeExtensionHelpers::CAN_USE_MJIT_HEADER
-  mjit_header_file_name = "rb_mjit_min_header-#{RUBY_VERSION}.h"
-
-  # Validate that the mjit header can actually be compiled on this system. We learned via
-  # https://github.com/DataDog/dd-trace-rb/issues/1799 and https://github.com/DataDog/dd-trace-rb/issues/1792
-  # that even if the header seems to exist, it may not even compile.
-  # `have_macro` actually tries to compile a file that mentions the given macro, so if this passes, we should be good to
-  # use the MJIT header.
-  # Finally, the `COMMON_HEADERS` conflict with the MJIT header so we need to temporarily disable them for this check.
-  original_common_headers = MakeMakefile::COMMON_HEADERS
-  MakeMakefile::COMMON_HEADERS = "".freeze
-  unless have_macro("RUBY_MJIT_H", mjit_header_file_name)
-    skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::COMPILATION_BROKEN)
-  end
-  MakeMakefile::COMMON_HEADERS = original_common_headers
-
-  $defs << "-DRUBY_MJIT_HEADER='\"#{mjit_header_file_name}\"'"
-
-  # NOTE: This needs to come after all changes to $defs
-  create_header
-
-  # Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
-  # See the comment on the same flag below for why this is done last.
-  append_cflags "-Wunused-parameter"
-
-  create_makefile EXTENSION_NAME
-else
-  # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  # the datadog-ruby_core_source gem to get access to private VM headers.
-  # This gem ships source code copies of these VM headers for the different Ruby VM versions;
-  # see https://github.com/DataDog/datadog-ruby_core_source for details
-
-  create_header
-
-  require "datadog/ruby_core_source"
-  dir_config("ruby") # allow user to pass in non-standard core include directory
-
-  # This is a workaround for a weird issue...
-  #
-  # The mkmf tool defines a `with_cppflags` helper that datadog-ruby_core_source uses. This helper temporarily
-  # replaces `$CPPFLAGS` (aka the C pre-processor [not c++!] flags) with a different set when doing something.
-  #
-  # The datadog-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
-  # path. But because `with_cppflags` replaces `$CPPFLAGS`, well, the default `$CPPFLAGS` are not included in the
-  # makefile.
-  #
-  # This is a problem because the default `$CPPFLAGS` carries configuration that was set when Ruby was being built.
-  # Thus, if we ignore it, we don't compile the profiler with the exact same configuration as Ruby.
-  # In practice, this can generate crashes and weird bugs if the Ruby configuration is tweaked in a manner that
-  # changes some of the internal structures that the profiler relies on. Concretely, setting for instance
-  # `VM_CHECK_MODE=1` when building Ruby will trigger this issue (because somethings in structures the profiler reads
-  # are ifdef'd out using this setting).
-  #
-  # To workaround this issue, we override `with_cppflags` for datadog-ruby_core_source to still include `$CPPFLAGS`.
-  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
-    super("#{newflags} #{$CPPFLAGS}", &block)
-  end
+# We rely on the datadog-ruby_core_source gem to get access to private VM headers.
+# This gem ships source code copies of these VM headers for the different Ruby VM versions;
+# see https://github.com/DataDog/datadog-ruby_core_source for details
 
-  Datadog::RubyCoreSource
-    .create_makefile_with_core(
-      proc do
-        headers_available =
-          have_header("vm_core.h") &&
-          have_header("iseq.h") &&
-          (RUBY_VERSION < "3.3" || have_header("ractor_core.h"))
-
-        if headers_available
-          # Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
-          # This is added as late as possible because in some Rubies we support (e.g. 3.3), adding this flag before
-          # checking if internal VM headers are available causes those checks to fail because of this warning (and not
-          # because the headers are not available.)
-          append_cflags "-Wunused-parameter"
-        end
-
-        headers_available
-      end,
-      EXTENSION_NAME,
-    )
+create_header
+
+require "datadog/ruby_core_source"
+dir_config("ruby") # allow user to pass in non-standard core include directory
+
+# This is a workaround for a weird issue...
+#
+# The mkmf tool defines a `with_cppflags` helper that datadog-ruby_core_source uses. This helper temporarily
+# replaces `$CPPFLAGS` (aka the C pre-processor [not c++!] flags) with a different set when doing something.
+#
+# The datadog-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
+# path. But because `with_cppflags` replaces `$CPPFLAGS`, well, the default `$CPPFLAGS` are not included in the
+# makefile.
+#
+# This is a problem because the default `$CPPFLAGS` carries configuration that was set when Ruby was being built.
+# Thus, if we ignore it, we don't compile the profiler with the exact same configuration as Ruby.
+# In practice, this can generate crashes and weird bugs if the Ruby configuration is tweaked in a manner that
+# changes some of the internal structures that the profiler relies on. Concretely, setting for instance
+# `VM_CHECK_MODE=1` when building Ruby will trigger this issue (because somethings in structures the profiler reads
+# are ifdef'd out using this setting).
+#
+# To workaround this issue, we override `with_cppflags` for datadog-ruby_core_source to still include `$CPPFLAGS`.
+Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+  # Note: The `Shellwords.escape` is an additional workaround for ruby_core_source supplying
+  # one unescaped -I argument containing the complete header path, not the CPPFLAGS thing above.
+  super("#{Shellwords.escape(newflags)} #{$CPPFLAGS}", &block)
 end
 
+Datadog::RubyCoreSource
+  .create_makefile_with_core(
+    proc do
+      headers_available =
+        have_header("vm_core.h") &&
+        have_header("iseq.h", "vm_core.h") &&
+        # These are only used on Ruby 3+
+        (RUBY_VERSION < "3" || have_header("ractor_core.h") && have_header("internal/class.h", "vm_core.h"))
+
+      if headers_available
+        # Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+        # This is added as late as possible because in some Rubies we support (e.g. 3.3), adding this flag before
+        # checking if internal VM headers are available causes those checks to fail because of this warning (and not
+        # because the headers are not available.)
+        append_cflags "-Wunused-parameter"
+      end
+
+      headers_available
+    end,
+    EXTENSION_NAME,
+  )
+
 # rubocop:enable Style/GlobalVars
 # rubocop:enable Style/StderrPuts

data/ext/datadog_profiling_native_extension/gvl_profiling_helper.c

@@ -4,49 +4,10 @@
 #include "datadog_ruby_common.h"
 #include "gvl_profiling_helper.h"
 
-#if !defined(NO_GVL_INSTRUMENTATION) && !defined(USE_GVL_PROFILING_3_2_WORKAROUNDS) // Ruby 3.3+
-  rb_internal_thread_specific_key_t gvl_waiting_tls_key;
+#ifdef HAVE_RUBY_THREAD_STORAGE_API
+  rb_internal_thread_specific_key_t per_thread_context_key;
 
-  void gvl_profiling_init(void) {
-    gvl_waiting_tls_key = rb_internal_thread_specific_key_create();
-  }
-
-#endif
-
-#ifdef USE_GVL_PROFILING_3_2_WORKAROUNDS // Ruby 3.2
-  __thread gvl_profiling_thread gvl_waiting_tls;
-  static bool gvl_profiling_state_thread_tracking_workaround_installed = false;
-
-  static void on_thread_start(
-    DDTRACE_UNUSED rb_event_flag_t _unused1,
-    DDTRACE_UNUSED const rb_internal_thread_event_data_t *_unused2,
-    DDTRACE_UNUSED void *_unused3
-  ) {
-    gvl_waiting_tls = (gvl_profiling_thread) {.thread = NULL};
-  }
-
-  // Hack: We're using the gvl_waiting_tls native thread-local to store per-thread information. Unfortunately, Ruby puts a big hole
-  // in our plan because it reuses native threads -- specifically, in Ruby 3.2, native threads are still 1:1 to Ruby
-  // threads (M:N wasn't a thing yet) BUT once a Ruby thread dies, the VM will keep the native thread around for a
-  // bit, and if another Ruby thread starts right after, Ruby will reuse the native thread, rather than create a new one.
-  //
-  // This will mean that the new Ruby thread will still have the same native thread-local data that we set on the
-  // old thread. For the purposes of our tracking, where we're keeping a pointer to the current thread object in
-  // thread-local storage **this is disastrous** since it means we'll be pointing at the wrong thread (and its
-  // memory may have been freed or reused since!)
-  //
-  // To work around this issue, once GVL profiling is enabled, we install an event hook on thread start
-  // events that clears the thread-local data. This guarantees that there will be no stale data -- any existing
-  // data will be cleared at thread start.
-  //
-  // Note that once installed, this event hook becomes permanent -- stopping the profiler does not stop this event
-  // hook, unlike all others. This is because we can't afford to miss any thread start events while the
-  // profiler is stopped (e.g. during reconfiguration) as that would mean stale data once the profiler starts again.
-  void gvl_profiling_state_thread_tracking_workaround(void) {
-    if (gvl_profiling_state_thread_tracking_workaround_installed) return;
-
-    rb_internal_thread_add_event_hook(on_thread_start, RUBY_INTERNAL_THREAD_EVENT_STARTED, NULL);
-
-    gvl_profiling_state_thread_tracking_workaround_installed = true;
+  void per_thread_context_tls_init(void) {
+    per_thread_context_key = rb_internal_thread_specific_key_create();
   }
 #endif

data/ext/datadog_profiling_native_extension/gvl_profiling_helper.h

@@ -6,62 +6,30 @@
 
 #include "extconf.h"
 
-#if !defined(NO_GVL_INSTRUMENTATION) && !defined(USE_GVL_PROFILING_3_2_WORKAROUNDS) // Ruby 3.3+
-  #include <ruby.h>
-  #include <ruby/thread.h>
-  #include "datadog_ruby_common.h"
+#include <ruby.h>
 
-  typedef struct { VALUE thread; } gvl_profiling_thread;
-  extern rb_internal_thread_specific_key_t gvl_waiting_tls_key;
+// Opaque to this header; the full definition lives in collectors_thread_context.c.
+typedef struct per_thread_context per_thread_context;
 
-  void gvl_profiling_init(void);
+#ifdef HAVE_RUBY_THREAD_STORAGE_API
+  #include <ruby/thread.h>
 
-  static inline gvl_profiling_thread thread_from_thread_object(VALUE thread) {
-    return (gvl_profiling_thread) {.thread = thread};
-  }
+  extern rb_internal_thread_specific_key_t per_thread_context_key;
 
-  static inline gvl_profiling_thread thread_from_event(const rb_internal_thread_event_data_t *event_data) {
-    return thread_from_thread_object(event_data->thread);
-  }
+  void per_thread_context_tls_init(void);
 
-  static inline intptr_t gvl_profiling_state_get(gvl_profiling_thread thread) {
-    return (intptr_t) rb_internal_thread_specific_get(thread.thread, gvl_waiting_tls_key);
+  static inline per_thread_context* get_per_thread_context(VALUE thread) {
+    return rb_internal_thread_specific_get(thread, per_thread_context_key);
   }
 
-  static inline void gvl_profiling_state_set(gvl_profiling_thread thread, intptr_t value) {
-    rb_internal_thread_specific_set(thread.thread, gvl_waiting_tls_key, (void *) value);
+  static inline void set_per_thread_context(VALUE thread, per_thread_context* value) {
+    rb_internal_thread_specific_set(thread, per_thread_context_key, value);
   }
-#endif
-
-#ifdef USE_GVL_PROFILING_3_2_WORKAROUNDS // Ruby 3.2
-  typedef struct { void *thread; } gvl_profiling_thread;
-  extern __thread gvl_profiling_thread gvl_waiting_tls;
-
-  static inline void gvl_profiling_init(void) { }
-
-  // NOTE: This is a hack that relies on the knowledge that on Ruby 3.2 the
-  // RUBY_INTERNAL_THREAD_EVENT_READY and RUBY_INTERNAL_THREAD_EVENT_RESUMED events always get called on the thread they
-  // are about. Thus, we can use our thread local storage hack to get this data, even though the event doesn't include it.
-  static inline gvl_profiling_thread thread_from_event(DDTRACE_UNUSED const void *event_data) {
-    return gvl_waiting_tls;
-  }
-
-  void gvl_profiling_state_thread_tracking_workaround(void);
-  gvl_profiling_thread gvl_profiling_state_maybe_initialize(void);
+#else
+  static inline void per_thread_context_tls_init(void) { }
 
   // Implementing these on Ruby 3.2 requires access to private VM things, so the following methods are
   // implemented in `private_vm_api_access.c`
-  gvl_profiling_thread thread_from_thread_object(VALUE thread);
-  intptr_t gvl_profiling_state_get(gvl_profiling_thread thread);
-  void gvl_profiling_state_set(gvl_profiling_thread thread, intptr_t value);
-#endif
-
-#ifndef NO_GVL_INSTRUMENTATION // For all Rubies supporting GVL profiling (3.2+)
-  static inline intptr_t gvl_profiling_state_thread_object_get(VALUE thread) {
-    return gvl_profiling_state_get(thread_from_thread_object(thread));
-  }
-
-  static inline void gvl_profiling_state_thread_object_set(VALUE thread, intptr_t value) {
-    gvl_profiling_state_set(thread_from_thread_object(thread), value);
-  }
+  per_thread_context* get_per_thread_context(VALUE thread);
+  void set_per_thread_context(VALUE thread, per_thread_context* value);
 #endif