datadog 2.3.0 → 2.5.0

data/ext/datadog_profiling_native_extension/heap_recorder.c

@@ -5,6 +5,7 @@
 #include <errno.h>
 #include "collectors_stack.h"
 #include "libdatadog_helpers.h"
+#include "time_helpers.h"
 
 #if (defined(HAVE_WORKING_RB_GC_FORCE_RECYCLE) && ! defined(NO_SEEN_OBJ_ID_FLAG))
   #define CAN_APPLY_GC_FORCE_RECYCLE_BUG_WORKAROUND
@@ -16,6 +17,16 @@
 // relevant for heap profiles as the great majority should be trivially reclaimed
 // during the next GC.
 #define ITERATION_MIN_AGE 1
+// Copied from https://github.com/ruby/ruby/blob/15135030e5808d527325feaaaf04caeb1b44f8b5/gc/default.c#L725C1-L725C27
+// to align with Ruby's GC definition of what constitutes an old object which are only
+// supposed to be reclaimed in major GCs.
+#define OLD_AGE 3
+// Wait at least 2 seconds before asking heap recorder to explicitly update itself. Heap recorder
+// data will only materialize at profile serialization time but updating often helps keep our
+// heap tracking data small since every GC should get rid of a bunch of temporary objects. The
+// more we clean up before profile flush, the less work we'll have to do all-at-once when preparing
+// to flush heap data and holding the GVL which should hopefully help with reducing latency impact.
+#define MIN_TIME_BETWEEN_HEAP_RECORDER_UPDATES_NS SECONDS_AS_NS(2)
 
 // A compact representation of a stacktrace frame for a heap allocation.
 typedef struct {
@@ -144,11 +155,18 @@ struct heap_recorder {
   // mutation of the data so iteration can occur without acquiring a lock.
   // NOTE: Contrary to object_records, this table has no ownership of its data.
   st_table *object_records_snapshot;
-  // The GC gen/epoch/count in which we prepared the current iteration.
+  // Are we currently updating or not?
+  bool updating;
+  // The GC gen/epoch/count in which we are updating (or last updated if not currently updating).
   //
-  // This enables us to calculate the age of iterated objects in the above snapshot by
-  // comparing it against an object's alloc_gen.
-  size_t iteration_gen;
+  // This enables us to calculate the age of objects considered in the update by comparing it
+  // against an object's alloc_gen.
+  size_t update_gen;
+  // Whether the current update (or last update if not currently updating) is including old
+  // objects or not.
+  bool update_include_old;
+  // When did we do the last update of heap recorder?
+  long last_update_ns;
 
   // Data for a heap recording that was started but not yet ended
   recording active_recording;
@@ -165,6 +183,21 @@ struct heap_recorder {
     size_t objects_skipped;
     size_t objects_frozen;
   } stats_last_update;
+
+  struct stats_lifetime {
+    unsigned long updates_successful;
+    unsigned long updates_skipped_concurrent;
+    unsigned long updates_skipped_gcgen;
+    unsigned long updates_skipped_time;
+
+    double ewma_young_objects_alive;
+    double ewma_young_objects_dead;
+    double ewma_young_objects_skipped; // Note: Here "young" refers to the young update; objects skipped includes non-young objects
+
+    double ewma_objects_alive;
+    double ewma_objects_dead;
+    double ewma_objects_skipped;
+  } stats_lifetime;
 };
 
 struct end_heap_allocation_args {
@@ -183,6 +216,8 @@ static int st_object_records_debug(st_data_t key, st_data_t value, st_data_t ext
 static int update_object_record_entry(st_data_t*, st_data_t*, st_data_t, int);
 static void commit_recording(heap_recorder*, heap_record*, recording);
 static VALUE end_heap_allocation_recording(VALUE end_heap_allocation_args);
+static void heap_recorder_update(heap_recorder *heap_recorder, bool full_update);
+static inline double ewma_stat(double previous, double current);
 
 // ==========================
 // Heap Recorder External API
@@ -280,6 +315,9 @@ void heap_recorder_after_fork(heap_recorder *heap_recorder) {
   if (heap_recorder->object_records_snapshot != NULL) {
     heap_recorder_finish_iteration(heap_recorder);
   }
+
+  // Clear lifetime stats since this is essentially a new heap recorder
+  heap_recorder->stats_lifetime = (struct stats_lifetime) {0};
 }
 
 void start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj, unsigned int weight, ddog_CharSlice *alloc_class) {
@@ -394,23 +432,94 @@ static VALUE end_heap_allocation_recording(VALUE end_heap_allocation_args) {
   return Qnil;
 }
 
-void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
+void heap_recorder_update_young_objects(heap_recorder *heap_recorder) {
   if (heap_recorder == NULL) {
     return;
   }
 
-  heap_recorder->iteration_gen = rb_gc_count();
+  heap_recorder_update(heap_recorder, /* full_update: */ false);
+}
+
+static void heap_recorder_update(heap_recorder *heap_recorder, bool full_update) {
+  if (heap_recorder->updating) {
+    if (full_update) rb_raise(rb_eRuntimeError, "BUG: full_update should not be triggered during another update");
+
+    // If we try to update while another update is still running, short-circuit.
+    // NOTE: This runs while holding the GVL. But since updates may be triggered from GC activity, there's still
+    //       a chance for updates to be attempted concurrently if scheduling gods so determine.
+    heap_recorder->stats_lifetime.updates_skipped_concurrent++;
+    return;
+  }
 
   if (heap_recorder->object_records_snapshot != NULL) {
-    // we could trivially handle this but we raise to highlight and catch unexpected usages.
-    rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
+    // While serialization is happening, it runs without the GVL and uses the object_records_snapshot.
+    // Although we iterate on a snapshot of object_records, these records point to other data that has not been
+    // snapshotted for efficiency reasons (e.g. heap_records). Since updating may invalidate
+    // some of that non-snapshotted data, let's refrain from doing updates during iteration. This also enforces the
+    // semantic that iteration will operate as a point-in-time snapshot.
+    return;
   }
 
+  size_t current_gc_gen = rb_gc_count();
+  long now_ns = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
+
+  if (!full_update) {
+    if (current_gc_gen == heap_recorder->update_gen) {
+      // Are we still in the same GC gen as last update? If so, skip updating since things should not have
+      // changed significantly since last time.
+      // NOTE: This is mostly a performance decision. I suppose some objects may be cleaned up in intermediate
+      // GC steps and sizes may change. But because we have to iterate through all our tracked
+      // object records to do an update, let's wait until all steps for a particular GC generation
+      // have finished to do so. We may revisit this once we have a better liveness checking mechanism.
+      heap_recorder->stats_lifetime.updates_skipped_gcgen++;
+      return;
+    }
+
+    if (now_ns > 0 && (now_ns - heap_recorder->last_update_ns) < MIN_TIME_BETWEEN_HEAP_RECORDER_UPDATES_NS) {
+      // We did an update not too long ago. Let's skip this one to avoid over-taxing the system.
+      heap_recorder->stats_lifetime.updates_skipped_time++;
+      return;
+    }
+  }
+
+  heap_recorder->updating = true;
   // Reset last update stats, we'll be building them from scratch during the st_foreach call below
-  heap_recorder->stats_last_update = (struct stats_last_update) {};
+  heap_recorder->stats_last_update = (struct stats_last_update) {0};
+
+  heap_recorder->update_gen = current_gc_gen;
+  heap_recorder->update_include_old = full_update;
 
   st_foreach(heap_recorder->object_records, st_object_record_update, (st_data_t) heap_recorder);
 
+  heap_recorder->last_update_ns = now_ns;
+  heap_recorder->stats_lifetime.updates_successful++;
+
+  // Lifetime stats updating
+  if (!full_update) {
+    heap_recorder->stats_lifetime.ewma_young_objects_alive = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_alive, heap_recorder->stats_last_update.objects_alive);
+    heap_recorder->stats_lifetime.ewma_young_objects_dead = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_dead, heap_recorder->stats_last_update.objects_dead);
+    heap_recorder->stats_lifetime.ewma_young_objects_skipped = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_skipped, heap_recorder->stats_last_update.objects_skipped);
+  } else {
+    heap_recorder->stats_lifetime.ewma_objects_alive = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_alive, heap_recorder->stats_last_update.objects_alive);
+    heap_recorder->stats_lifetime.ewma_objects_dead = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_dead, heap_recorder->stats_last_update.objects_dead);
+    heap_recorder->stats_lifetime.ewma_objects_skipped = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_skipped, heap_recorder->stats_last_update.objects_skipped);
+  }
+
+  heap_recorder->updating = false;
+}
+
+void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
+  if (heap_recorder == NULL) {
+    return;
+  }
+
+  if (heap_recorder->object_records_snapshot != NULL) {
+    // we could trivially handle this but we raise to highlight and catch unexpected usages.
+    rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
+  }
+
+  heap_recorder_update(heap_recorder, /* full_update: */ true);
+
   heap_recorder->object_records_snapshot = st_copy(heap_recorder->object_records);
   if (heap_recorder->object_records_snapshot == NULL) {
     rb_raise(rb_eRuntimeError, "Failed to create heap snapshot.");
@@ -474,6 +583,19 @@ VALUE heap_recorder_state_snapshot(heap_recorder *heap_recorder) {
     ID2SYM(rb_intern("last_update_objects_dead")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_dead),
     ID2SYM(rb_intern("last_update_objects_skipped")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_skipped),
     ID2SYM(rb_intern("last_update_objects_frozen")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_frozen),
+
+    // Lifetime stats
+    ID2SYM(rb_intern("lifetime_updates_successful")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_successful),
+    ID2SYM(rb_intern("lifetime_updates_skipped_concurrent")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_concurrent),
+    ID2SYM(rb_intern("lifetime_updates_skipped_gcgen")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_gcgen),
+    ID2SYM(rb_intern("lifetime_updates_skipped_time")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_time),
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_alive")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_alive),
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_dead")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_dead),
+      // Note: Here "young" refers to the young update; objects skipped includes non-young objects
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_skipped")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_skipped),
+    ID2SYM(rb_intern("lifetime_ewma_objects_alive")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_alive),
+    ID2SYM(rb_intern("lifetime_ewma_objects_dead")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_dead),
+    ID2SYM(rb_intern("lifetime_ewma_objects_skipped")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_skipped),
   };
   VALUE hash = rb_hash_new();
   for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(hash, arguments[i], arguments[i+1]);
@@ -503,11 +625,14 @@ void heap_recorder_testonly_assert_hash_matches(ddog_prof_Slice_Location locatio
 
 VALUE heap_recorder_testonly_debug(heap_recorder *heap_recorder) {
   if (heap_recorder == NULL) {
-    return rb_str_new2("NULL heap_recorder");
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
   }
 
   VALUE debug_str = rb_str_new2("object records:\n");
   st_foreach(heap_recorder->object_records, st_object_records_debug, (st_data_t) debug_str);
+
+  rb_str_catf(debug_str, "state snapshot: %"PRIsVALUE"\n------\n", heap_recorder_state_snapshot(heap_recorder));
+
   return debug_str;
 }
 
@@ -526,13 +651,6 @@ static int st_object_record_entry_free(DDTRACE_UNUSED st_data_t key, st_data_t v
   return ST_DELETE;
 }
 
-// Check to see if an object should not be included in a heap recorder iteration.
-// This centralizes the checking logic to ensure it's equally applied between
-// preparation and iteration codepaths.
-static inline bool should_exclude_from_iteration(object_record *obj_record) {
-  return obj_record->object_data.gen_age < ITERATION_MIN_AGE;
-}
-
 static int st_object_record_update(st_data_t key, st_data_t value, st_data_t extra_arg) {
   long obj_id = (long) key;
   object_record *record = (object_record*) value;
@@ -540,16 +658,20 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
 
   VALUE ref;
 
-  size_t iteration_gen = recorder->iteration_gen;
+  size_t update_gen = recorder->update_gen;
   size_t alloc_gen = record->object_data.alloc_gen;
   // Guard against potential overflows given unsigned types here.
-  record->object_data.gen_age = alloc_gen < iteration_gen ? iteration_gen - alloc_gen : 0;
+  record->object_data.gen_age = alloc_gen < update_gen ? update_gen - alloc_gen : 0;
+
+  if (record->object_data.gen_age == 0) {
+    // Objects that belong to the current GC gen have not had a chance to be cleaned up yet
+    // and won't show up in the iteration anyway so no point in checking their liveness/sizes.
+    recorder->stats_last_update.objects_skipped++;
+    return ST_CONTINUE;
+  }
 
-  if (should_exclude_from_iteration(record)) {
-    // If an object won't be included in the current iteration, there's
-    // no point checking for liveness or updating its size, so exit early.
-    // NOTE: This means that there should be an equivalent check during actual
-    //       iteration otherwise we'd iterate/expose stale object data.
+  if (!recorder->update_include_old && record->object_data.gen_age >= OLD_AGE) {
+    // The current update is not including old objects but this record is for an old object, skip its update.
     recorder->stats_last_update.objects_skipped++;
     return ST_CONTINUE;
   }
@@ -598,7 +720,11 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
 
   #endif
 
-  if (recorder->size_enabled && !record->object_data.is_frozen) {
+  if (
+    recorder->size_enabled &&
+    recorder->update_include_old && // We only update sizes when doing a full update
+    !record->object_data.is_frozen
+  ) {
     // if we were asked to update sizes and this object was not already seen as being frozen,
     // update size again.
     record->object_data.size = ruby_obj_memsize_of(ref);
@@ -622,22 +748,22 @@ static int st_object_records_iterate(DDTRACE_UNUSED st_data_t key, st_data_t val
 
   const heap_recorder *recorder = context->heap_recorder;
 
-  if (should_exclude_from_iteration(record)) {
+  if (record->object_data.gen_age < ITERATION_MIN_AGE) {
     // Skip objects that should not be included in iteration
-    // NOTE: This matches the short-circuiting condition in st_object_record_update
-    //       and prevents iteration over stale objects.
     return ST_CONTINUE;
   }
 
   ddog_prof_Location *locations = recorder->reusable_locations;
   for (uint16_t i = 0; i < stack->frames_len; i++) {
     const heap_frame *frame = &stack->frames[i];
-    ddog_prof_Location *location = &locations[i];
-    location->function.name.ptr = frame->name;
-    location->function.name.len = strlen(frame->name);
-    location->function.filename.ptr = frame->filename;
-    location->function.filename.len = strlen(frame->filename);
-    location->line = frame->line;
+    locations[i] = (ddog_prof_Location) {
+      .mapping = {.filename = DDOG_CHARSLICE_C(""), .build_id = DDOG_CHARSLICE_C("")},
+      .function = {
+        .name = {.ptr = frame->name, .len = strlen(frame->name)},
+        .filename = {.ptr = frame->filename, .len = strlen(frame->filename)},
+      },
+      .line = frame->line,
+    };
   }
 
   heap_recorder_iteration_data iteration_data;
@@ -782,8 +908,20 @@ static void cleanup_heap_record_if_unused(heap_recorder *heap_recorder, heap_rec
 }
 
 static void on_committed_object_record_cleanup(heap_recorder *heap_recorder, object_record *record) {
+  // @ivoanjo: We've seen a segfault crash in the field in this function (October 2024) which we're still trying to investigate.
+  // (See PROF-10656 Datadog-internal for details). Just in case, I've sprinkled a bunch of NULL tests in this function for now.
+  // Once we figure out the issue we can get rid of them again.
+
+  if (heap_recorder == NULL) rb_raise(rb_eRuntimeError, "heap_recorder was NULL in on_committed_object_record_cleanup");
+  if (heap_recorder->heap_records == NULL) rb_raise(rb_eRuntimeError, "heap_recorder->heap_records was NULL in on_committed_object_record_cleanup");
+  if (record == NULL) rb_raise(rb_eRuntimeError, "record was NULL in on_committed_object_record_cleanup");
+
   // Starting with the associated heap record. There will now be one less tracked object pointing to it
   heap_record *heap_record = record->heap_record;
+
+  if (heap_record == NULL) rb_raise(rb_eRuntimeError, "heap_record was NULL in on_committed_object_record_cleanup");
+  if (heap_record->stack == NULL) rb_raise(rb_eRuntimeError, "heap_record->stack was NULL in on_committed_object_record_cleanup");
+
   heap_record->num_tracked_objects--;
 
   // One less object using this heap record, it may have become unused...
@@ -1073,3 +1211,25 @@ st_index_t heap_record_key_hash_st(st_data_t key) {
     return ddog_location_slice_hash(*record_key->location_slice, FNV1_32A_INIT);
   }
 }
+
+static inline double ewma_stat(double previous, double current) {
+  double alpha = 0.3;
+  return (1 - alpha) * previous + alpha * current;
+}
+
+VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, VALUE obj_id) {
+  if (heap_recorder == NULL) {
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
+  }
+
+  // Check if object records contains an object with this object_id
+  return st_is_member(heap_recorder->object_records, FIX2LONG(obj_id)) ? Qtrue : Qfalse;
+}
+
+void heap_recorder_testonly_reset_last_update(heap_recorder *heap_recorder) {
+  if (heap_recorder == NULL) {
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
+  }
+
+  heap_recorder->last_update_ns = 0;
+}

data/ext/datadog_profiling_native_extension/heap_recorder.h

@@ -118,6 +118,11 @@ void start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj
 __attribute__((warn_unused_result))
 int end_heap_allocation_recording_with_rb_protect(heap_recorder *heap_recorder, ddog_prof_Slice_Location locations);
 
+// Update the heap recorder, **checking young objects only**. The idea here is to align with GC: most young objects never
+// survive enough GC generations, and thus periodically running this method reduces memory usage (we get rid of
+// these objects quicker) and hopefully reduces tail latency (because there's less objects at serialization time to check).
+void heap_recorder_update_young_objects(heap_recorder *heap_recorder);
+
 // Update the heap recorder to reflect the latest state of the VM and prepare internal structures
 // for efficient iteration.
 //
@@ -166,3 +171,9 @@ void heap_recorder_testonly_assert_hash_matches(ddog_prof_Slice_Location locatio
 // Returns a Ruby string with a representation of internal data helpful to
 // troubleshoot issues such as unexpected test failures.
 VALUE heap_recorder_testonly_debug(heap_recorder *heap_recorder);
+
+// Check if a given object_id is being tracked or not
+VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, VALUE obj_id);
+
+// Used to ensure that a GC actually triggers an update of the objects
+void heap_recorder_testonly_reset_last_update(heap_recorder *heap_recorder);

data/ext/datadog_profiling_native_extension/http_transport.c

@@ -77,6 +77,32 @@ static VALUE _native_validate_exporter(DDTRACE_UNUSED VALUE _self, VALUE exporte
   return rb_ary_new_from_args(2, ok_symbol, Qnil);
 }
 
+static ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
+  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
+
+  VALUE exporter_working_mode = rb_ary_entry(exporter_configuration, 0);
+  ENFORCE_TYPE(exporter_working_mode, T_SYMBOL);
+  ID working_mode = SYM2ID(exporter_working_mode);
+
+  ID agentless_id = rb_intern("agentless");
+  ID agent_id = rb_intern("agent");
+
+  if (working_mode != agentless_id && working_mode != agent_id) {
+    rb_raise(rb_eArgError, "Failed to initialize transport: Unexpected working mode, expected :agentless or :agent");
+  }
+
+  if (working_mode == agentless_id) {
+    VALUE site = rb_ary_entry(exporter_configuration, 1);
+    VALUE api_key = rb_ary_entry(exporter_configuration, 2);
+
+    return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
+  } else { // agent_id
+    VALUE base_url = rb_ary_entry(exporter_configuration, 1);
+
+    return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
+  }
+}
+
 static ddog_prof_Exporter_NewResult create_exporter(VALUE exporter_configuration, VALUE tags_as_array) {
   ENFORCE_TYPE(exporter_configuration, T_ARRAY);
   ENFORCE_TYPE(tags_as_array, T_ARRAY);
@@ -115,8 +141,7 @@ static VALUE perform_export(
   ddog_prof_Exporter_Slice_File files_to_export_unmodified,
   ddog_Vec_Tag *additional_tags,
   ddog_CharSlice internal_metadata,
-  ddog_CharSlice info,
-  uint64_t timeout_milliseconds
+  ddog_CharSlice info
 ) {
   ddog_prof_ProfiledEndpointsStats *endpoints_stats = NULL; // Not in use yet
   ddog_prof_Exporter_Request_BuildResult build_result = ddog_prof_Exporter_Request_build(
@@ -128,8 +153,7 @@ static VALUE perform_export(
     additional_tags,
     endpoints_stats,
     &internal_metadata,
-    &info,
-    timeout_milliseconds
+    &info
   );
 
   if (build_result.tag == DDOG_PROF_EXPORTER_REQUEST_BUILD_RESULT_ERR) {
@@ -254,6 +278,15 @@ static VALUE _native_do_export(
   VALUE failure_tuple = handle_exporter_failure(exporter_result);
   if (!NIL_P(failure_tuple)) return failure_tuple;
 
+  ddog_prof_MaybeError timeout_result = ddog_prof_Exporter_set_timeout(exporter_result.ok, timeout_milliseconds);
+  if (timeout_result.tag == DDOG_PROF_OPTION_ERROR_SOME_ERROR) {
+    // NOTE: Seems a bit harsh to fail the upload if we can't set a timeout. OTOH, this is only expected to fail
+    // if the exporter is not well built. Because such a situation should already be caught above I think it's
+    // preferable to leave this here as a virtually unreachable exception rather than ignoring it.
+    ddog_prof_Exporter_drop(exporter_result.ok);
+    return rb_ary_new_from_args(2, error_symbol, get_error_details_and_drop(&timeout_result.some));
+  }
+
   return perform_export(
     exporter_result.ok,
     start,
@@ -262,8 +295,7 @@ static VALUE _native_do_export(
     files_to_export_unmodified,
     null_additional_tags,
     internal_metadata,
-    info,
-    timeout_milliseconds
+    info
   );
 }
 

data/ext/datadog_profiling_native_extension/native_extension_helpers.rb

@@ -9,7 +9,7 @@ module Datadog
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = "DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION"
 
-      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
+      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on datadog-ruby_core_source
       CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
 
       def self.fail_install_if_missing_extension?

data/ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -13,7 +13,7 @@
   #include RUBY_MJIT_HEADER
 #else
   // The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  // the debase-ruby_core_source gem to get access to private VM headers.
+  // the datadog-ruby_core_source gem to get access to private VM headers.
 
   // We can't do anything about warnings in VM headers, so we just use this technique to suppress them.
   // See https://nelkinda.com/blog/suppress-warnings-in-gcc-and-clang/#d11e364 for details.
@@ -182,7 +182,7 @@ uint64_t native_thread_id_for(VALUE thread) {
   #if !defined(NO_THREAD_TID) && defined(RB_THREAD_T_HAS_NATIVE_ID)
     #ifndef NO_RB_NATIVE_THREAD
       struct rb_native_thread* native_thread = thread_struct_from_object(thread)->nt;
-      if (native_thread == NULL) rb_raise(rb_eRuntimeError, "BUG: rb_native_thread* is null. Is this Ruby running with RUBY_MN_THREADS=1?");
+      if (native_thread == NULL) return 0;
       return native_thread->tid;
     #else
       return thread_struct_from_object(thread)->tid;
@@ -755,3 +755,54 @@ static inline int ddtrace_imemo_type(VALUE imemo) {
     return GET_VM()->objspace;
   }
 #endif
+
+#ifdef USE_GVL_PROFILING_3_2_WORKAROUNDS // Ruby 3.2
+  #include "gvl_profiling_helper.h"
+
+  gvl_profiling_thread thread_from_thread_object(VALUE thread) {
+    return (gvl_profiling_thread) {.thread = thread_struct_from_object(thread)};
+  }
+
+  // Hack: In Ruby 3.3+ we attach gvl profiling state to Ruby threads using the
+  // rb_internal_thread_specific_* APIs. These APIs did not exist on Ruby 3.2. On Ruby 3.2 we instead store the
+  // needed data inside the `rb_thread_t` structure, specifically in `stat_insn_usage` as a Ruby FIXNUM.
+  //
+  // Why `stat_insn_usage`? We needed some per-thread storage, and while looking at the Ruby VM sources I noticed
+  // that `stat_insn_usage` has been in `rb_thread_t` for a long time, but is not used anywhere in the VM
+  // code. There's a comment attached to it "/* statistics data for profiler */" but other than marking this
+  // field for GC, I could not find any place in the VM commit history or on GitHub where this has ever been used.
+  //
+  // Thus, since this hack is only for 3.2, which presumably will never see this field either removed or used
+  // during its remaining maintenance release period we... kinda take it for our own usage. It's ugly, I know...
+  intptr_t gvl_profiling_state_get(gvl_profiling_thread thread) {
+    if (thread.thread == NULL) return 0;
+
+    VALUE current_value = ((rb_thread_t *)thread.thread)->stat_insn_usage;
+    intptr_t result = current_value == Qnil ? 0 : FIX2LONG(current_value);
+    return result;
+  }
+
+  void gvl_profiling_state_set(gvl_profiling_thread thread, intptr_t value) {
+    if (thread.thread == NULL) return;
+    ((rb_thread_t *)thread.thread)->stat_insn_usage = LONG2FIX(value);
+  }
+
+  // Because Ruby 3.2 does not give us the current thread when calling the RUBY_INTERNAL_THREAD_EVENT_READY and
+  // RUBY_INTERNAL_THREAD_EVENT_RESUMED APIs, we need to figure out this info ourselves.
+  //
+  // Specifically, this method was created to be called from a RUBY_INTERNAL_THREAD_EVENT_RESUMED callback --
+  // when it's triggered, we know the thread the code gets executed on is holding the GVL, so we use this
+  // opportunity to initialize our thread-local value.
+  gvl_profiling_thread gvl_profiling_state_maybe_initialize(void) {
+    gvl_profiling_thread current_thread = gvl_waiting_tls;
+
+    if (current_thread.thread == NULL) {
+      // threads.sched.running is the thread currently holding the GVL, which when this gets executed is the
+      // current thread!
+      current_thread = (gvl_profiling_thread) {.thread = (void *) rb_current_ractor()->threads.sched.running};
+      gvl_waiting_tls = current_thread;
+    }
+
+    return current_thread;
+  }
+#endif

data/ext/datadog_profiling_native_extension/private_vm_api_access.h

@@ -65,3 +65,6 @@ const char *imemo_kind(VALUE imemo);
 #ifdef NO_POSTPONED_TRIGGER
   void *objspace_ptr_for_gc_finalize_deferred_workaround(void);
 #endif
+
+#define ENFORCE_THREAD(value) \
+  { if (RB_UNLIKELY(!rb_typeddata_is_kind_of(value, RTYPEDDATA_TYPE(rb_thread_current())))) raise_unexpected_type(value, ADD_QUOTES(value), "Thread", __FILE__, __LINE__, __func__); }

data/ext/datadog_profiling_native_extension/profiling.c

@@ -253,7 +253,7 @@ static VALUE _native_enforce_success(DDTRACE_UNUSED VALUE _self, VALUE syserr_er
 
 static void *trigger_enforce_success(void *trigger_args) {
   intptr_t syserr_errno = (intptr_t) trigger_args;
-  ENFORCE_SUCCESS_NO_GVL(syserr_errno);
+  ENFORCE_SUCCESS_NO_GVL((int) syserr_errno);
   return NULL;
 }
 

data/ext/datadog_profiling_native_extension/ruby_helpers.c

@@ -219,16 +219,19 @@ static bool ruby_is_obj_with_class(VALUE obj) {
   return false;
 }
 
-VALUE ruby_safe_inspect(VALUE obj) {
-  if (!ruby_is_obj_with_class(obj)) {
-    return rb_str_new_cstr("(Not an object)");
-  }
+// These two functions are not present in the VM headers, but are public symbols that can be invoked.
+int rb_objspace_internal_object_p(VALUE obj);
+const char *rb_obj_info(VALUE obj);
 
-  if (rb_respond_to(obj, inspect_id)) {
-    return rb_sprintf("%+"PRIsVALUE, obj);
-  } else if (rb_respond_to(obj, to_s_id)) {
-    return rb_sprintf("%"PRIsVALUE, obj);
-  } else {
-    return rb_str_new_cstr("(Not inspectable)");
-  }
+VALUE ruby_safe_inspect(VALUE obj) {
+  if (!ruby_is_obj_with_class(obj))       return rb_str_new_cstr("(Not an object)");
+  if (rb_objspace_internal_object_p(obj)) return rb_sprintf("(VM Internal, %s)", rb_obj_info(obj));
+  // @ivoanjo: I saw crashes on Ruby 3.1.4 when trying to #inspect matchdata objects. I'm not entirely sure why this
+  // is needed, but since we only use this method for debug purposes I put in this alternative and decided not to
+  // dig deeper.
+  if (rb_type(obj) == RUBY_T_MATCH)   return rb_sprintf("(MatchData, %s)", rb_obj_info(obj));
+  if (rb_respond_to(obj, inspect_id)) return rb_sprintf("%+"PRIsVALUE, obj);
+  if (rb_respond_to(obj, to_s_id))    return rb_sprintf("%"PRIsVALUE, obj);
+
+  return rb_str_new_cstr("(Not inspectable)");
 }