datadog 2.2.0 → 2.3.0

data/ext/datadog_profiling_native_extension/collectors_thread_context.c

@@ -92,7 +92,8 @@ struct thread_context_collector_state {
   // "Update this when modifying state struct"
 
   // Required by Datadog::Profiling::Collectors::Stack as a scratch buffer during sampling
-  sampling_buffer *sampling_buffer;
+  ddog_prof_Location *locations;
+  uint16_t max_frames;
   // Hashmap <Thread Object, struct per_thread_context>
   st_table *hash_map_per_thread_context;
   // Datadog::Profiling::StackRecorder instance
@@ -138,6 +139,7 @@ struct thread_context_collector_state {
 
 // Tracks per-thread state
 struct per_thread_context {
+  sampling_buffer *sampling_buffer;
   char thread_id[THREAD_ID_LIMIT_CHARS];
   ddog_CharSlice thread_id_char_slice;
   char thread_invoke_location[THREAD_INVOKE_LOCATION_LIMIT_CHARS];
@@ -184,8 +186,9 @@ static VALUE _native_sample_after_gc(DDTRACE_UNUSED VALUE self, VALUE collector_
 void update_metrics_and_sample(
   struct thread_context_collector_state *state,
   VALUE thread_being_sampled,
-  VALUE profiler_overhead_stack_thread,
+  VALUE stack_from_thread,
   struct per_thread_context *thread_context,
+  sampling_buffer* sampling_buffer,
   long current_cpu_time_ns,
   long current_monotonic_wall_time_ns
 );
@@ -194,6 +197,7 @@ static void trigger_sample_for_thread(
   VALUE thread,
   VALUE stack_from_thread,
   struct per_thread_context *thread_context,
+  sampling_buffer* sampling_buffer,
   sample_values values,
   long current_monotonic_wall_time_ns,
   ddog_CharSlice *ruby_vm_type,
@@ -203,6 +207,7 @@ static VALUE _native_thread_list(VALUE self);
 static struct per_thread_context *get_or_create_context_for(VALUE thread, struct thread_context_collector_state *state);
 static struct per_thread_context *get_context_for(VALUE thread, struct thread_context_collector_state *state);
 static void initialize_context(VALUE thread, struct per_thread_context *thread_context, struct thread_context_collector_state *state);
+static void free_context(struct per_thread_context* thread_context);
 static VALUE _native_inspect(VALUE self, VALUE collector_instance);
 static VALUE per_thread_context_st_table_as_ruby_hash(struct thread_context_collector_state *state);
 static int per_thread_context_as_ruby_hash(st_data_t key_thread, st_data_t value_context, st_data_t result_hash);
@@ -231,6 +236,7 @@ static void ddtrace_otel_trace_identifiers_for(
   VALUE active_span,
   VALUE otel_values
 );
+static VALUE _native_sample_skipped_allocation_samples(DDTRACE_UNUSED VALUE self, VALUE collector_instance, VALUE skipped_samples);
 
 void collectors_thread_context_init(VALUE profiling_module) {
   VALUE collectors_module = rb_define_module_under(profiling_module, "Collectors");
@@ -261,6 +267,7 @@ void collectors_thread_context_init(VALUE profiling_module) {
   rb_define_singleton_method(testing_module, "_native_stats", _native_stats, 1);
   rb_define_singleton_method(testing_module, "_native_gc_tracking", _native_gc_tracking, 1);
   rb_define_singleton_method(testing_module, "_native_new_empty_thread", _native_new_empty_thread, 0);
+  rb_define_singleton_method(testing_module, "_native_sample_skipped_allocation_samples", _native_sample_skipped_allocation_samples, 2);
 
   at_active_span_id = rb_intern_const("@active_span");
   at_active_trace_id = rb_intern_const("@active_trace");
@@ -308,7 +315,7 @@ static void thread_context_collector_typed_data_free(void *state_ptr) {
 
   // Important: Remember that we're only guaranteed to see here what's been set in _native_new, aka
   // pointers that have been set NULL there may still be NULL here.
-  if (state->sampling_buffer != NULL) sampling_buffer_free(state->sampling_buffer);
+  if (state->locations != NULL) ruby_xfree(state->locations);
 
   // Free each entry in the map
   st_foreach(state->hash_map_per_thread_context, hash_map_per_thread_context_free_values, 0 /* unused */);
@@ -327,8 +334,8 @@ static int hash_map_per_thread_context_mark(st_data_t key_thread, DDTRACE_UNUSED
 
 // Used to clear each of the per_thread_contexts inside the hash_map_per_thread_context
 static int hash_map_per_thread_context_free_values(DDTRACE_UNUSED st_data_t _thread, st_data_t value_per_thread_context, DDTRACE_UNUSED st_data_t _argument) {
-  struct per_thread_context *per_thread_context = (struct per_thread_context*) value_per_thread_context;
-  ruby_xfree(per_thread_context);
+  struct per_thread_context *thread_context = (struct per_thread_context*) value_per_thread_context;
+  free_context(thread_context);
   return ST_CONTINUE;
 }
 
@@ -339,23 +346,35 @@ static VALUE _native_new(VALUE klass) {
   // being leaked.
 
   // Update this when modifying state struct
-  state->sampling_buffer = NULL;
+  state->locations = NULL;
+  state->max_frames = 0;
   state->hash_map_per_thread_context =
    // "numtable" is an awful name, but TL;DR it's what should be used when keys are `VALUE`s.
     st_init_numtable();
   state->recorder_instance = Qnil;
   state->tracer_context_key = MISSING_TRACER_CONTEXT_KEY;
-  state->thread_list_buffer = rb_ary_new();
+  VALUE thread_list_buffer = rb_ary_new();
+  state->thread_list_buffer = thread_list_buffer;
   state->endpoint_collection_enabled = true;
   state->timeline_enabled = true;
   state->allocation_type_enabled = true;
   state->time_converter_state = (monotonic_to_system_epoch_state) MONOTONIC_TO_SYSTEM_EPOCH_INITIALIZER;
-  state->main_thread = rb_thread_main();
+  VALUE main_thread = rb_thread_main();
+  state->main_thread = main_thread;
   state->otel_current_span_key = Qnil;
   state->gc_tracking.wall_time_at_previous_gc_ns = INVALID_TIME;
   state->gc_tracking.wall_time_at_last_flushed_gc_event_ns = 0;
 
-  return TypedData_Wrap_Struct(klass, &thread_context_collector_typed_data, state);
+  // Note: Remember to keep any new allocated objects that get stored in the state also on the stack + mark them with
+  // RB_GC_GUARD -- otherwise it's possible for a GC to run and
+  // since the instance representing the state does not yet exist, such objects will not get marked.
+
+  VALUE instance = TypedData_Wrap_Struct(klass, &thread_context_collector_typed_data, state);
+
+  RB_GC_GUARD(thread_list_buffer);
+  RB_GC_GUARD(main_thread); // Arguably not needed, but perhaps can be move in some future Ruby release?
+
+  return instance;
 }
 
 static VALUE _native_initialize(
@@ -375,11 +394,9 @@ static VALUE _native_initialize(
   struct thread_context_collector_state *state;
   TypedData_Get_Struct(collector_instance, struct thread_context_collector_state, &thread_context_collector_typed_data, state);
 
-  int max_frames_requested = NUM2INT(max_frames);
-  if (max_frames_requested < 0) rb_raise(rb_eArgError, "Invalid max_frames: value must not be negative");
-
   // Update this when modifying state struct
-  state->sampling_buffer = sampling_buffer_new(max_frames_requested);
+  state->max_frames = sampling_buffer_check_max_frames(NUM2INT(max_frames));
+  state->locations = ruby_xcalloc(state->max_frames, sizeof(ddog_prof_Location));
   // hash_map_per_thread_context is already initialized, nothing to do here
   state->recorder_instance = enforce_recorder_instance(recorder_instance);
   state->endpoint_collection_enabled = (endpoint_collection_enabled == Qtrue);
@@ -461,6 +478,7 @@ void thread_context_collector_sample(VALUE self_instance, long current_monotonic
       /* thread_being_sampled: */ thread,
       /* stack_from_thread: */ thread,
       thread_context,
+      thread_context->sampling_buffer,
       current_cpu_time_ns,
       current_monotonic_wall_time_ns
     );
@@ -477,6 +495,8 @@ void thread_context_collector_sample(VALUE self_instance, long current_monotonic
     /* thread_being_sampled: */ current_thread,
     /* stack_from_thread: */ profiler_overhead_stack_thread,
     current_thread_context,
+    // Here we use the overhead thread's sampling buffer so as to not invalidate the cache in the buffer of the thread being sampled
+    get_or_create_context_for(profiler_overhead_stack_thread, state)->sampling_buffer,
     cpu_time_now_ns(current_thread_context),
     monotonic_wall_time_now_ns(RAISE_ON_FAILURE)
   );
@@ -487,6 +507,7 @@ void update_metrics_and_sample(
   VALUE thread_being_sampled,
   VALUE stack_from_thread, // This can be different when attributing profiler overhead using a different stack
   struct per_thread_context *thread_context,
+  sampling_buffer* sampling_buffer,
   long current_cpu_time_ns,
   long current_monotonic_wall_time_ns
 ) {
@@ -512,6 +533,7 @@ void update_metrics_and_sample(
     thread_being_sampled,
     stack_from_thread,
     thread_context,
+    sampling_buffer,
     (sample_values) {.cpu_time_ns = cpu_time_elapsed_ns, .cpu_or_wall_samples = 1, .wall_time_ns = wall_time_elapsed_ns},
     current_monotonic_wall_time_ns,
     NULL,
@@ -661,7 +683,6 @@ VALUE thread_context_collector_sample_after_gc(VALUE self_instance) {
   }
 
   record_placeholder_stack(
-    state->sampling_buffer,
     state->recorder_instance,
     (sample_values) {
       // This event gets both a regular cpu/wall-time duration, as a normal cpu/wall-time sample would, as well as a
@@ -692,6 +713,7 @@ static void trigger_sample_for_thread(
   VALUE thread,
   VALUE stack_from_thread, // This can be different when attributing profiler overhead using a different stack
   struct per_thread_context *thread_context,
+  sampling_buffer* sampling_buffer,
   sample_values values,
   long current_monotonic_wall_time_ns,
   // These two labels are only used for allocation profiling; @ivoanjo: may want to refactor this at some point?
@@ -812,7 +834,7 @@ static void trigger_sample_for_thread(
 
   sample_thread(
     stack_from_thread,
-    state->sampling_buffer,
+    sampling_buffer,
     state->recorder_instance,
     values,
     (sample_labels) {.labels = slice_labels, .state_label = state_label, .end_timestamp_ns = end_timestamp_ns}
@@ -875,6 +897,8 @@ static bool is_logging_gem_monkey_patch(VALUE invoke_file_location) {
 }
 
 static void initialize_context(VALUE thread, struct per_thread_context *thread_context, struct thread_context_collector_state *state) {
+  thread_context->sampling_buffer = sampling_buffer_new(state->max_frames, state->locations);
+
   snprintf(thread_context->thread_id, THREAD_ID_LIMIT_CHARS, "%"PRIu64" (%lu)", native_thread_id_for(thread), (unsigned long) thread_id_for(thread));
   thread_context->thread_id_char_slice = (ddog_CharSlice) {.ptr = thread_context->thread_id, .len = strlen(thread_context->thread_id)};
 
@@ -915,6 +939,11 @@ static void initialize_context(VALUE thread, struct per_thread_context *thread_c
   thread_context->gc_tracking.wall_time_at_start_ns = INVALID_TIME;
 }
 
+static void free_context(struct per_thread_context* thread_context) {
+  sampling_buffer_free(thread_context->sampling_buffer);
+  ruby_xfree(thread_context);
+}
+
 static VALUE _native_inspect(DDTRACE_UNUSED VALUE _self, VALUE collector_instance) {
   struct thread_context_collector_state *state;
   TypedData_Get_Struct(collector_instance, struct thread_context_collector_state, &thread_context_collector_typed_data, state);
@@ -922,6 +951,7 @@ static VALUE _native_inspect(DDTRACE_UNUSED VALUE _self, VALUE collector_instanc
   VALUE result = rb_str_new2(" (native state)");
 
   // Update this when modifying state struct
+  rb_str_concat(result, rb_sprintf(" max_frames=%d", state->max_frames));
   rb_str_concat(result, rb_sprintf(" hash_map_per_thread_context=%"PRIsVALUE, per_thread_context_st_table_as_ruby_hash(state)));
   rb_str_concat(result, rb_sprintf(" recorder_instance=%"PRIsVALUE, state->recorder_instance));
   VALUE tracer_context_key = state->tracer_context_key == MISSING_TRACER_CONTEXT_KEY ? Qnil : ID2SYM(state->tracer_context_key);
@@ -1006,7 +1036,7 @@ static int remove_if_dead_thread(st_data_t key_thread, st_data_t value_context,
 
   if (is_thread_alive(thread)) return ST_CONTINUE;
 
-  ruby_xfree(thread_context);
+  free_context(thread_context);
   return ST_DELETE;
 }
 
@@ -1257,7 +1287,7 @@ void thread_context_collector_sample_allocation(VALUE self_instance, unsigned in
       // Thus, we need to make sure there's actually a class before getting its name.
 
       if (klass != 0) {
-        const char *name = rb_obj_classname(new_object);
+        const char *name = rb_class2name(klass);
         size_t name_length = name != NULL ? strlen(name) : 0;
 
         if (name_length > 0) {
@@ -1285,12 +1315,15 @@ void thread_context_collector_sample_allocation(VALUE self_instance, unsigned in
 
   track_object(state->recorder_instance, new_object, sample_weight, optional_class_name);
 
+  struct per_thread_context *thread_context = get_or_create_context_for(current_thread, state);
+
   trigger_sample_for_thread(
     state,
     /* thread: */  current_thread,
     /* stack_from_thread: */ current_thread,
-    get_or_create_context_for(current_thread, state),
-    (sample_values) {.alloc_samples = sample_weight, .alloc_samples_unscaled = 1},
+    thread_context,
+    thread_context->sampling_buffer,
+    (sample_values) {.alloc_samples = sample_weight, .alloc_samples_unscaled = 1, .heap_sample = true},
     INVALID_TIME, // For now we're not collecting timestamps for allocation events, as per profiling team internal discussions
     &ruby_vm_type,
     optional_class_name
@@ -1400,3 +1433,32 @@ static void ddtrace_otel_trace_identifiers_for(
   *active_trace = current_trace;
   *numeric_span_id = resolved_numeric_span_id;
 }
+
+void thread_context_collector_sample_skipped_allocation_samples(VALUE self_instance, unsigned int skipped_samples) {
+  struct thread_context_collector_state *state;
+  TypedData_Get_Struct(self_instance, struct thread_context_collector_state, &thread_context_collector_typed_data, state);
+
+  ddog_prof_Label labels[] = {
+    // Providing .num = 0 should not be needed but the tracer-2.7 docker image ships a buggy gcc that complains about this
+    {.key = DDOG_CHARSLICE_C("thread id"),        .str = DDOG_CHARSLICE_C("SS"),                .num = 0},
+    {.key = DDOG_CHARSLICE_C("thread name"),      .str = DDOG_CHARSLICE_C("Skipped Samples"),   .num = 0},
+    {.key = DDOG_CHARSLICE_C("allocation class"), .str = DDOG_CHARSLICE_C("(Skipped Samples)"), .num = 0},
+  };
+  ddog_prof_Slice_Label slice_labels = {.ptr = labels, .len = sizeof(labels) / sizeof(labels[0])};
+
+  record_placeholder_stack(
+    state->recorder_instance,
+    (sample_values) {.alloc_samples = skipped_samples},
+    (sample_labels) {
+      .labels = slice_labels,
+      .state_label = NULL,
+      .end_timestamp_ns = 0, // For now we're not collecting timestamps for allocation events
+    },
+    DDOG_CHARSLICE_C("Skipped Samples")
+  );
+}
+
+static VALUE _native_sample_skipped_allocation_samples(DDTRACE_UNUSED VALUE self, VALUE collector_instance, VALUE skipped_samples) {
+  thread_context_collector_sample_skipped_allocation_samples(collector_instance, NUM2UINT(skipped_samples));
+  return Qtrue;
+}

data/ext/datadog_profiling_native_extension/collectors_thread_context.h

@@ -9,6 +9,7 @@ void thread_context_collector_sample(
   VALUE profiler_overhead_stack_thread
 );
 void thread_context_collector_sample_allocation(VALUE self_instance, unsigned int sample_weight, VALUE new_object);
+void thread_context_collector_sample_skipped_allocation_samples(VALUE self_instance, unsigned int skipped_samples);
 VALUE thread_context_collector_sample_after_gc(VALUE self_instance);
 void thread_context_collector_on_gc_start(VALUE self_instance);
 bool thread_context_collector_on_gc_finish(VALUE self_instance);

data/ext/datadog_profiling_native_extension/datadog_ruby_common.c

@@ -0,0 +1,110 @@
+#include "datadog_ruby_common.h"
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name) {
+  rb_exc_raise(
+    rb_exc_new_str(
+      rb_eTypeError,
+      rb_sprintf("wrong argument %"PRIsVALUE" for '%s' (expected a %s) at %s:%d:in `%s'",
+        rb_inspect(value),
+        value_name,
+        type_name,
+        file,
+        line,
+        function_name
+      )
+    )
+  );
+}
+
+VALUE datadog_gem_version(void) {
+  VALUE ddtrace_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  ENFORCE_TYPE(ddtrace_module, T_MODULE);
+  VALUE version_module = rb_const_get(ddtrace_module, rb_intern("VERSION"));
+  ENFORCE_TYPE(version_module, T_MODULE);
+  VALUE version_string = rb_const_get(version_module, rb_intern("STRING"));
+  ENFORCE_TYPE(version_string, T_STRING);
+  return version_string;
+}
+
+__attribute__((warn_unused_result))
+ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
+  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
+
+  VALUE exporter_working_mode = rb_ary_entry(exporter_configuration, 0);
+  ENFORCE_TYPE(exporter_working_mode, T_SYMBOL);
+  ID working_mode = SYM2ID(exporter_working_mode);
+
+  ID agentless_id = rb_intern("agentless");
+  ID agent_id = rb_intern("agent");
+
+  if (working_mode != agentless_id && working_mode != agent_id) {
+    rb_raise(rb_eArgError, "Failed to initialize transport: Unexpected working mode, expected :agentless or :agent");
+  }
+
+  if (working_mode == agentless_id) {
+    VALUE site = rb_ary_entry(exporter_configuration, 1);
+    VALUE api_key = rb_ary_entry(exporter_configuration, 2);
+
+    return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
+  } else { // agent_id
+    VALUE base_url = rb_ary_entry(exporter_configuration, 1);
+
+    return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
+  }
+}
+
+static VALUE log_failure_to_process_tag(VALUE err_details) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE logger = rb_funcall(datadog_module, rb_intern("logger"), 0);
+
+  return rb_funcall(logger, rb_intern("warn"), 1, rb_sprintf("Failed to convert tag: %"PRIsVALUE, err_details));
+}
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
+  ENFORCE_TYPE(tags_as_array, T_ARRAY);
+
+  long tags_count = RARRAY_LEN(tags_as_array);
+  ddog_Vec_Tag tags = ddog_Vec_Tag_new();
+
+  for (long i = 0; i < tags_count; i++) {
+    VALUE name_value_pair = rb_ary_entry(tags_as_array, i);
+
+    if (!RB_TYPE_P(name_value_pair, T_ARRAY)) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(name_value_pair, T_ARRAY);
+    }
+
+    // Note: We can index the array without checking its size first because rb_ary_entry returns Qnil if out of bounds
+    VALUE tag_name = rb_ary_entry(name_value_pair, 0);
+    VALUE tag_value = rb_ary_entry(name_value_pair, 1);
+
+    if (!(RB_TYPE_P(tag_name, T_STRING) && RB_TYPE_P(tag_value, T_STRING))) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(tag_name, T_STRING);
+      ENFORCE_TYPE(tag_value, T_STRING);
+    }
+
+    ddog_Vec_Tag_PushResult push_result =
+      ddog_Vec_Tag_push(&tags, char_slice_from_ruby_string(tag_name), char_slice_from_ruby_string(tag_value));
+
+    if (push_result.tag == DDOG_VEC_TAG_PUSH_RESULT_ERR) {
+      // libdatadog validates tags and may catch invalid tags that ddtrace didn't actually catch.
+      // We warn users about such tags, and then just ignore them.
+
+      int exception_state;
+      rb_protect(log_failure_to_process_tag, get_error_details_and_drop(&push_result.err), &exception_state);
+
+      // Since we are calling into Ruby code, it may raise an exception. Ensure that dynamically-allocated tags
+      // get cleaned before propagating the exception.
+      if (exception_state) {
+        ddog_Vec_Tag_drop(tags);
+        rb_jump_tag(exception_state);  // "Re-raise" exception
+      }
+    }
+  }
+
+  return tags;
+}

data/ext/datadog_profiling_native_extension/datadog_ruby_common.h

@@ -0,0 +1,57 @@
+#pragma once
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+#include <ruby.h>
+#include <datadog/profiling.h>
+
+// Used to mark symbols to be exported to the outside of the extension.
+// Consider very carefully before tagging a function with this.
+#define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
+
+// Used to mark function arguments that are deliberately left unused
+#ifdef __GNUC__
+  #define DDTRACE_UNUSED  __attribute__((unused))
+#else
+  #define DDTRACE_UNUSED
+#endif
+
+#define ADD_QUOTES_HELPER(x) #x
+#define ADD_QUOTES(x) ADD_QUOTES_HELPER(x)
+
+// Ruby has a Check_Type(value, type) that is roughly equivalent to this BUT Ruby's version is rather cryptic when it fails
+// e.g. "wrong argument type nil (expected String)". This is a replacement that prints more information to help debugging.
+#define ENFORCE_TYPE(value, type) \
+  { if (RB_UNLIKELY(!RB_TYPE_P(value, type))) raise_unexpected_type(value, ADD_QUOTES(value), ADD_QUOTES(type), __FILE__, __LINE__, __func__); }
+
+#define ENFORCE_BOOLEAN(value) \
+  { if (RB_UNLIKELY(value != Qtrue && value != Qfalse)) raise_unexpected_type(value, ADD_QUOTES(value), "true or false", __FILE__, __LINE__, __func__); }
+
+// Called by ENFORCE_TYPE; should not be used directly
+NORETURN(void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name));
+
+// Helper to retrieve Datadog::VERSION::STRING
+VALUE datadog_gem_version(void);
+
+static inline ddog_CharSlice char_slice_from_ruby_string(VALUE string) {
+  ENFORCE_TYPE(string, T_STRING);
+  ddog_CharSlice char_slice = {.ptr = RSTRING_PTR(string), .len = RSTRING_LEN(string)};
+  return char_slice;
+}
+
+__attribute__((warn_unused_result))
+ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration);
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array);
+
+static inline VALUE ruby_string_from_error(const ddog_Error *error) {
+  ddog_CharSlice char_slice = ddog_Error_message(error);
+  return rb_str_new(char_slice.ptr, char_slice.len);
+}
+
+static inline VALUE get_error_details_and_drop(ddog_Error *error) {
+  VALUE result = ruby_string_from_error(error);
+  ddog_Error_drop(error);
+  return result;
+}