datadog 2.21.0 → 2.38.0

data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.c

@@ -6,7 +6,7 @@
 #include "ruby_helpers.h"
 
 #define BASE_OVERHEAD_PCT 1.0
-#define BASE_SAMPLING_INTERVAL 50
+#define BASE_SAMPLING_INTERVAL 1
 
 #define ADJUSTMENT_WINDOW_NS SECONDS_AS_NS(1)
 #define ADJUSTMENT_WINDOW_SAMPLES 100
@@ -51,7 +51,7 @@ void discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, long now_
 
 void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead, long now_ns) {
   if (target_overhead <= 0 || target_overhead > 100) {
-    rb_raise(rb_eArgError, "Target overhead must be a double between ]0,100] was %f", target_overhead);
+    raise_error(rb_eArgError, "Target overhead must be a double between ]0,100] was %f", target_overhead);
   }
   sampler->target_overhead = target_overhead;
   return discrete_dynamic_sampler_reset(sampler, now_ns);
@@ -365,11 +365,11 @@ static const rb_data_type_t sampler_typed_data = {
 };
 
 static VALUE _native_new(VALUE klass) {
-  sampler_state *state = ruby_xcalloc(sizeof(sampler_state), 1);
+  sampler_state *state = ruby_xcalloc(1, sizeof(sampler_state));
 
   long now_ns = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
   if (now_ns == 0) {
-    rb_raise(rb_eRuntimeError, "failed to get clock time");
+    raise_error(rb_eRuntimeError, "failed to get clock time");
   }
   discrete_dynamic_sampler_init(&state->sampler, "test sampler", now_ns);
 

data/ext/datadog_profiling_native_extension/collectors_gc_profiling_helper.c

@@ -2,6 +2,7 @@
 #include <datadog/profiling.h>
 
 #include "collectors_gc_profiling_helper.h"
+#include "ruby_helpers.h"
 
 // This helper is used by the Datadog::Profiling::Collectors::ThreadContext to profile garbage collection.
 // It's tested through that class' interfaces.
@@ -71,7 +72,7 @@ uint8_t gc_profiling_set_metadata(ddog_prof_Label *labels, int labels_length) {
     1;  // gc type
 
   if (max_label_count > labels_length) {
-    rb_raise(rb_eArgError, "BUG: gc_profiling_set_metadata invalid labels_length (%d) < max_label_count (%d)", labels_length, max_label_count);
+    raise_error(rb_eArgError, "BUG: gc_profiling_set_metadata invalid labels_length (%d) < max_label_count (%d)", labels_length, max_label_count);
   }
 
   uint8_t label_pos = 0;
@@ -119,7 +120,7 @@ uint8_t gc_profiling_set_metadata(ddog_prof_Label *labels, int labels_length) {
   };
 
   if (label_pos > max_label_count) {
-    rb_raise(rb_eRuntimeError, "BUG: gc_profiling_set_metadata unexpected label_pos (%d) > max_label_count (%d)", label_pos, max_label_count);
+    raise_error(rb_eRuntimeError, "BUG: gc_profiling_set_metadata unexpected label_pos (%d) > max_label_count (%d)", label_pos, max_label_count);
   }
 
   return label_pos;

data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c

@@ -6,6 +6,7 @@
 #include "helpers.h"
 #include "ruby_helpers.h"
 #include "collectors_idle_sampling_helper.h"
+#include "collectors_thread_context.h"
 
 // Used by the Collectors::CpuAndWallTimeWorker to gather samples when the Ruby process is idle.
 //
@@ -30,7 +31,9 @@ typedef struct {
 
 static VALUE _native_new(VALUE klass);
 static void reset_state(idle_sampling_loop_state *state);
-static VALUE _native_idle_sampling_loop(DDTRACE_UNUSED VALUE self, VALUE self_instance);
+static VALUE _native_idle_sampling_loop(DDTRACE_UNUSED VALUE self, VALUE self_instance, VALUE thread_context_collector_instance);
+static VALUE idle_sampling_loop_body(VALUE self_instance);
+static VALUE idle_sampling_loop_ensure(VALUE thread_context_collector_instance);
 static VALUE _native_stop(DDTRACE_UNUSED VALUE self, VALUE self_instance);
 static void *run_idle_sampling_loop(void *state_ptr);
 static void interrupt_idle_sampling_loop(void *state_ptr);
@@ -56,7 +59,7 @@ void collectors_idle_sampling_helper_init(VALUE profiling_module) {
   // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
   rb_define_alloc_func(collectors_idle_sampling_helper_class, _native_new);
 
-  rb_define_singleton_method(collectors_idle_sampling_helper_class, "_native_idle_sampling_loop", _native_idle_sampling_loop, 1);
+  rb_define_singleton_method(collectors_idle_sampling_helper_class, "_native_idle_sampling_loop", _native_idle_sampling_loop, 2);
   rb_define_singleton_method(collectors_idle_sampling_helper_class, "_native_stop", _native_stop, 1);
   rb_define_singleton_method(collectors_idle_sampling_helper_class, "_native_reset", _native_reset, 1);
   rb_define_singleton_method(testing_module, "_native_idle_sampling_helper_request_action", _native_idle_sampling_helper_request_action, 1);
@@ -109,16 +112,27 @@ static VALUE _native_reset(DDTRACE_UNUSED VALUE self, VALUE self_instance) {
   return Qtrue;
 }
 
-static VALUE _native_idle_sampling_loop(DDTRACE_UNUSED VALUE self, VALUE self_instance) {
+static VALUE _native_idle_sampling_loop(DDTRACE_UNUSED VALUE self, VALUE self_instance, VALUE thread_context_collector_instance) {
+  return rb_ensure(idle_sampling_loop_body, self_instance, idle_sampling_loop_ensure, thread_context_collector_instance);
+}
+
+static VALUE idle_sampling_loop_body(VALUE self_instance) {
   idle_sampling_loop_state *state;
   TypedData_Get_Struct(self_instance, idle_sampling_loop_state, &idle_sampling_helper_typed_data, state);
 
+  thread_context_collector_profiler_internal_thread_started();
+
   // Release GVL and run the loop waiting for requests
   rb_thread_call_without_gvl(run_idle_sampling_loop, state, interrupt_idle_sampling_loop, state);
 
   return Qtrue;
 }
 
+static VALUE idle_sampling_loop_ensure(VALUE thread_context_collector_instance) {
+  thread_context_collector_profiler_internal_thread_done(thread_context_collector_instance);
+  return Qnil;
+}
+
 static void *run_idle_sampling_loop(void *state_ptr) {
   idle_sampling_loop_state *state = (idle_sampling_loop_state *) state_ptr;
   int error = 0;
@@ -208,7 +222,7 @@ void idle_sampling_helper_request_action(VALUE self_instance, void (*run_action_
   if (!rb_typeddata_is_kind_of(self_instance, &idle_sampling_helper_typed_data)) {
     grab_gvl_and_raise(rb_eTypeError, "Wrong argument for idle_sampling_helper_request_action");
   }
-  // This should never fail the the above check passes
+  // This should never fail when the above check passes
   TypedData_Get_Struct(self_instance, idle_sampling_loop_state, &idle_sampling_helper_typed_data, state);
 
   ENFORCE_SUCCESS_NO_GVL(pthread_mutex_lock(&state->wakeup_mutex));

data/ext/datadog_profiling_native_extension/collectors_stack.c

@@ -17,6 +17,7 @@
 
 #include "datadog_ruby_common.h"
 #include "private_vm_api_access.h"
+#include "ruby_helpers.h"
 #include "stack_recorder.h"
 #include "collectors_stack.h"
 
@@ -39,7 +40,7 @@ static void set_file_info_for_cfunc(
   st_table *native_filenames_cache
 );
 static const char *get_or_compute_native_filename(void *function, st_table *native_filenames_cache);
-static void add_truncated_frames_placeholder(sampling_buffer* buffer);
+static void add_truncated_frames_placeholder(ddog_prof_Location *locations);
 static void record_placeholder_stack_in_native_code(VALUE recorder_instance, sample_values values, sample_labels labels);
 static void maybe_trim_template_random_ids(ddog_CharSlice *name_slice, ddog_CharSlice *filename_slice);
 
@@ -97,7 +98,7 @@ typedef struct {
   sample_values values;
   sample_labels labels;
   VALUE thread;
-  ddog_prof_Location *locations;
+  sample_locations locations;
   sampling_buffer *buffer;
   bool native_filenames_enabled;
   st_table *native_filenames_cache;
@@ -174,7 +175,7 @@ static VALUE _native_sample(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
 
   ddog_prof_Location *locations = ruby_xcalloc(max_frames_requested, sizeof(ddog_prof_Location));
   sampling_buffer buffer;
-  sampling_buffer_initialize(&buffer, max_frames_requested, locations);
+  sampling_buffer_initialize(&buffer, max_frames_requested);
 
   ddog_prof_Slice_Label slice_labels = {.ptr = labels, .len = labels_count};
 
@@ -184,7 +185,7 @@ static VALUE _native_sample(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
     .values = values,
     .labels = (sample_labels) {.labels = slice_labels, .state_label = state_label, .is_gvl_waiting_state = is_gvl_waiting_state == Qtrue},
     .thread = thread,
-    .locations = locations,
+    .locations = (sample_locations) {.ptr = locations, .len = max_frames_requested},
     .buffer = &buffer,
     .native_filenames_enabled = native_filenames_enabled == Qtrue,
     .native_filenames_cache = st_init_numtable(),
@@ -207,6 +208,7 @@ static VALUE native_sample_do(VALUE args) {
     sample_thread(
       args_struct->thread,
       args_struct->buffer,
+      args_struct->locations,
       args_struct->recorder_instance,
       args_struct->values,
       args_struct->labels,
@@ -221,7 +223,7 @@ static VALUE native_sample_do(VALUE args) {
 static VALUE native_sample_ensure(VALUE args) {
   native_sample_args *args_struct = (native_sample_args *) args;
 
-  ruby_xfree(args_struct->locations);
+  ruby_xfree(args_struct->locations.ptr);
   sampling_buffer_free(args_struct->buffer);
   st_free_table(args_struct->native_filenames_cache);
 
@@ -242,14 +244,29 @@ static VALUE native_sample_ensure(VALUE args) {
 void sample_thread(
   VALUE thread,
   sampling_buffer* buffer,
+  sample_locations locations,
   VALUE recorder_instance,
   sample_values values,
   sample_labels labels,
   bool native_filenames_enabled,
   st_table *native_filenames_cache
 ) {
+  if (buffer->max_frames != locations.len) {
+    // This shouldn't happen as thread_context_collector_reset_all_per_thread_contexts (which resizes every
+    // per-thread sampling buffer to match the collector's max_frames) must always be called before starting or
+    // restarting profiling.
+    raise_error(
+      rb_eRuntimeError,
+      "Unexpected: sampling buffer max_frames (%d) doesn't match locations len (%d)",
+      (int) buffer->max_frames,
+      (int) locations.len
+    );
+  }
+
   // If we already prepared a sample, we use it below; if not, we prepare it now.
-  if (!buffer->pending_sample) prepare_sample_thread(thread, buffer);
+  if (!buffer->pending_sample) {
+    prepare_sample_thread(thread, buffer);
+  }
 
   buffer->pending_sample = false;
   int captured_frames = buffer->pending_sample_result;
@@ -284,11 +301,11 @@ void sample_thread(
   // here, but >= 0 makes this easier to understand/debug.
   bool only_wall_time = cpu_or_wall_sample && values.cpu_time_ns == 0 && values.wall_time_ns >= 0;
 
-  if (cpu_or_wall_sample && state_label == NULL) rb_raise(rb_eRuntimeError, "BUG: Unexpected missing state_label");
+  if (cpu_or_wall_sample && state_label == NULL) raise_error(rb_eRuntimeError, "BUG: Unexpected missing state_label");
 
   if (has_cpu_time) {
     state_label->str = DDOG_CHARSLICE_C("had cpu");
-    if (labels.is_gvl_waiting_state) rb_raise(rb_eRuntimeError, "BUG: Unexpected combination of cpu-time with is_gvl_waiting");
+    if (labels.is_gvl_waiting_state) raise_error(rb_eRuntimeError, "BUG: Unexpected combination of cpu-time with is_gvl_waiting");
   }
 
   int top_of_stack_position = captured_frames - 1;
@@ -299,8 +316,13 @@ void sample_thread(
     bool top_of_the_stack = i == top_of_stack_position;
 
     if (buffer->stack_buffer[i].is_ruby_frame) {
-      VALUE name = rb_iseq_base_label(buffer->stack_buffer[i].as.ruby_frame.iseq);
-      VALUE filename = rb_iseq_path(buffer->stack_buffer[i].as.ruby_frame.iseq);
+      VALUE iseq = buffer->stack_buffer[i].as.ruby_frame.iseq;
+      VALUE name = rb_iseq_base_label(iseq);
+      VALUE filename =
+        // Note: We saw crash reports from a customer from dereferencing a null pointer inside `rb_iseq_path`.
+        // It's not clear in what situation would the pathobj ever be NULL, yet it seems harmless from our side to
+        // skip this information if we're ever in such a situation.
+        pathobj_is_null(iseq) ? Qnil : rb_iseq_path(iseq);
 
       name_slice = NIL_P(name) ? DDOG_CHARSLICE_C("") : char_slice_from_ruby_string(name);
       filename_slice = NIL_P(filename) ? DDOG_CHARSLICE_C("") : char_slice_from_ruby_string(filename);
@@ -347,13 +369,19 @@ void sample_thread(
         } else if (CHARSLICE_EQUALS("select", name_slice)) { // Expected to be Kernel.select
           state_label->str  = DDOG_CHARSLICE_C("waiting");
         } else if (
-            CHARSLICE_EQUALS("synchronize", name_slice) || // Expected to be Monitor/Mutex#synchronize
-            CHARSLICE_EQUALS("lock", name_slice) ||        // Expected to be Mutex#lock
+            CHARSLICE_EQUALS("synchronize", name_slice) || // Expected to be Monitor/Mutex#synchronize on Ruby 2 & 3, and Monitor#synchronize on 4 (Mutex becomes <internal:thread_sync>)
+            #ifdef NO_PRIMITIVE_MUTEX_AND_CONDITION_VARIABLE // Ruby < 4
+              CHARSLICE_EQUALS("lock", name_slice) ||        // Expected to be Mutex#lock
+            #endif
             CHARSLICE_EQUALS("join", name_slice)           // Expected to be Thread#join
         ) {
           state_label->str  = DDOG_CHARSLICE_C("blocked");
         } else if (CHARSLICE_EQUALS("wait_readable", name_slice)) { // Expected to be IO#wait_readable
           state_label->str  = DDOG_CHARSLICE_C("network");
+        } else if (CHARSLICE_EQUALS("_native_idle_sampling_loop", name_slice)) { // Expected to be Datadog::Profiler::Collectors::IdleSamplingHelper#_native_idle_sampling_loop
+          state_label->str  = DDOG_CHARSLICE_C("waiting");
+        } else if (CHARSLICE_EQUALS("_native_sampling_loop", name_slice)) { // Expected to be Datadog::Profiler::Collectors::CpuAndWallTimeWorker#_native_sampling_loop
+          state_label->str  = DDOG_CHARSLICE_C("sleeping");
         }
         #ifdef NO_PRIMITIVE_POP // Ruby < 3.2
           else if (CHARSLICE_EQUALS("pop", name_slice)) { // Expected to be Queue/SizedQueue#pop
@@ -362,9 +390,19 @@ void sample_thread(
         #endif
       } else {
         #ifndef NO_PRIMITIVE_POP // Ruby >= 3.2
-          // Unlike the above, Ruby actually treats this one specially and gives it a nice file name we can match on!
-          if (CHARSLICE_EQUALS("pop", name_slice) && CHARSLICE_EQUALS("<internal:thread_sync>", filename_slice)) { // Expected to be Queue/SizedQueue#pop
-            state_label->str  = DDOG_CHARSLICE_C("waiting");
+          if (CHARSLICE_EQUALS("<internal:thread_sync>", filename_slice)) {
+            if (CHARSLICE_EQUALS("pop", name_slice)) { // Expected to be Queue/SizedQueue#pop
+              state_label->str  = DDOG_CHARSLICE_C("waiting");
+            }
+            #ifndef NO_PRIMITIVE_MUTEX_AND_CONDITION_VARIABLE // Ruby >= 4
+              else if (CHARSLICE_EQUALS("synchronize", name_slice) || CHARSLICE_EQUALS("lock", name_slice)) { // Expected to be Mutex#lock/synchronize
+                state_label->str  = DDOG_CHARSLICE_C("blocked");
+              } else if (CHARSLICE_EQUALS("sleep", name_slice)) { // Expected to be Mutex#sleep
+                state_label->str  = DDOG_CHARSLICE_C("sleeping");
+              } else if (CHARSLICE_EQUALS("wait", name_slice)) { // Expected to be ConditionVariable#wait
+                state_label->str  = DDOG_CHARSLICE_C("waiting");
+              }
+            #endif
           }
         #endif
       }
@@ -372,22 +410,22 @@ void sample_thread(
 
     int libdatadog_stores_stacks_flipped_from_rb_profile_frames_index = top_of_stack_position - i;
 
-    buffer->locations[libdatadog_stores_stacks_flipped_from_rb_profile_frames_index] = (ddog_prof_Location) {
+    locations.ptr[libdatadog_stores_stacks_flipped_from_rb_profile_frames_index] = (ddog_prof_Location) {
       .mapping = {.filename = DDOG_CHARSLICE_C(""), .build_id = DDOG_CHARSLICE_C(""), .build_id_id = {}},
       .function = (ddog_prof_Function) {.name = name_slice, .filename = filename_slice},
       .line = line,
     };
   }
 
-  // If we filled up the buffer, some frames may have been omitted. In that case, we'll add a placeholder frame
+  // If we filled up the locations, some frames may have been omitted. In that case, we'll add a placeholder frame
   // with that info.
-  if (captured_frames == (long) buffer->max_frames) {
-    add_truncated_frames_placeholder(buffer);
+  if (captured_frames == (long) locations.len) {
+    add_truncated_frames_placeholder(locations.ptr);
   }
 
   record_sample(
     recorder_instance,
-    (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = captured_frames},
+    (ddog_prof_Slice_Location) {.ptr = locations.ptr, .len = captured_frames},
     values,
     labels
   );
@@ -521,10 +559,10 @@ static void maybe_trim_template_random_ids(ddog_CharSlice *name_slice, ddog_Char
   name_slice->len = pos;
 }
 
-static void add_truncated_frames_placeholder(sampling_buffer* buffer) {
+static void add_truncated_frames_placeholder(ddog_prof_Location *locations) {
   // Important note: The strings below are static so we don't need to worry about their lifetime. If we ever want to change
   // this to non-static strings, don't forget to check that lifetimes are properly respected.
-  buffer->locations[0] = (ddog_prof_Location) {
+  locations[0] = (ddog_prof_Location) {
     .mapping = {.filename = DDOG_CHARSLICE_C(""), .build_id = DDOG_CHARSLICE_C(""), .build_id_id = {}},
     .function = {.name = DDOG_CHARSLICE_C("Truncated Frames"), .filename = DDOG_CHARSLICE_C(""), .filename_id = {}},
     .line = 0,
@@ -596,16 +634,15 @@ bool prepare_sample_thread(VALUE thread, sampling_buffer *buffer) {
 }
 
 uint16_t sampling_buffer_check_max_frames(int max_frames) {
-  if (max_frames < 5) rb_raise(rb_eArgError, "Invalid max_frames: value must be >= 5");
-  if (max_frames > MAX_FRAMES_LIMIT) rb_raise(rb_eArgError, "Invalid max_frames: value must be <= " MAX_FRAMES_LIMIT_AS_STRING);
+  if (max_frames < 5) raise_error(rb_eArgError, "Invalid max_frames: value must be >= 5");
+  if (max_frames > MAX_FRAMES_LIMIT) raise_error(rb_eArgError, "Invalid max_frames: value must be <= " MAX_FRAMES_LIMIT_AS_STRING);
   return max_frames;
 }
 
-void sampling_buffer_initialize(sampling_buffer *buffer, uint16_t max_frames, ddog_prof_Location *locations) {
+void sampling_buffer_initialize(sampling_buffer *buffer, uint16_t max_frames) {
   sampling_buffer_check_max_frames(max_frames);
 
   buffer->max_frames = max_frames;
-  buffer->locations = locations;
   buffer->stack_buffer = ruby_xcalloc(max_frames, sizeof(frame_info));
   buffer->pending_sample = false;
   buffer->is_marking = false;
@@ -613,15 +650,9 @@ void sampling_buffer_initialize(sampling_buffer *buffer, uint16_t max_frames, dd
 }
 
 void sampling_buffer_free(sampling_buffer *buffer) {
-  if (buffer->max_frames == 0 || buffer->locations == NULL || buffer->stack_buffer == NULL) {
-    rb_raise(rb_eArgError, "sampling_buffer_free called with invalid buffer");
-  }
-
   ruby_xfree(buffer->stack_buffer);
-  // Note: buffer->locations are owned by whoever called sampling_buffer_initialize, not by the buffer itself
 
   buffer->max_frames = 0;
-  buffer->locations = NULL;
   buffer->stack_buffer = NULL;
   buffer->pending_sample = false;
   buffer->is_marking = false;

data/ext/datadog_profiling_native_extension/collectors_stack.h

@@ -11,16 +11,21 @@
 // Used as scratch space during sampling
 typedef struct {
   uint16_t max_frames;
-  ddog_prof_Location *locations;
   frame_info *stack_buffer;
   bool pending_sample;
   bool is_marking; // Used to avoid recording a sample when marking
   int pending_sample_result;
 } sampling_buffer;
 
+typedef struct {
+  ddog_prof_Location *ptr;
+  uint16_t len;
+} sample_locations;
+
 void sample_thread(
   VALUE thread,
   sampling_buffer* buffer,
+  sample_locations locations,
   VALUE recorder_instance,
   sample_values values,
   sample_labels labels,
@@ -36,7 +41,7 @@ void record_placeholder_stack(
 bool prepare_sample_thread(VALUE thread, sampling_buffer *buffer);
 
 uint16_t sampling_buffer_check_max_frames(int max_frames);
-void sampling_buffer_initialize(sampling_buffer *buffer, uint16_t max_frames, ddog_prof_Location *locations);
+void sampling_buffer_initialize(sampling_buffer *buffer, uint16_t max_frames);
 void sampling_buffer_free(sampling_buffer *buffer);
 void sampling_buffer_mark(sampling_buffer *buffer);
 static inline bool sampling_buffer_needs_marking(sampling_buffer *buffer) {