ddtrace 1.18.0 → 1.23.2

data/ext/datadog_profiling_native_extension/collectors_gc_profiling_helper.c

@@ -0,0 +1,156 @@
+#include <ruby.h>
+#include <datadog/profiling.h>
+
+#include "collectors_gc_profiling_helper.h"
+
+// This helper is used by the Datadog::Profiling::Collectors::ThreadContext to profile garbage collection.
+// It's tested through that class' interfaces.
+// ---
+
+// Used when retrieving GC information from the VM.
+//  All these are symbols, but we don't need to mark them since we ask for them to be interned (and thus live forever)
+static VALUE state_sym;
+static VALUE marking_sym;
+static VALUE sweeping_sym;
+static VALUE none_sym;
+static VALUE gc_by_sym;
+static VALUE newobj_sym;
+static VALUE malloc_sym;
+static VALUE method_sym;
+static VALUE capi_sym;
+static VALUE stress_sym;
+static VALUE major_by_sym;
+static VALUE nofree_sym;
+static VALUE oldgen_sym;
+static VALUE shady_sym;
+static VALUE force_sym;
+static VALUE oldmalloc_sym;
+
+static ddog_CharSlice major_gc_reason_pretty(VALUE major_gc_reason);
+static ddog_CharSlice gc_cause_pretty(VALUE gc_cause);
+static ddog_CharSlice gc_type_pretty(VALUE major_gc_reason, VALUE gc_state);
+
+void gc_profiling_init(void) {
+  // This function lazy-interns a few constants, which may trigger allocations. Since we want to call it during GC as
+  // well, when allocations are not allowed, we call it once here so that the constants get defined ahead of time.
+  rb_gc_latest_gc_info(rb_hash_new());
+
+  // Used to query and look up the results of GC information
+  state_sym     = ID2SYM(rb_intern_const("state"));
+  marking_sym   = ID2SYM(rb_intern_const("marking"));
+  sweeping_sym  = ID2SYM(rb_intern_const("sweeping"));
+  none_sym      = ID2SYM(rb_intern_const("none"));
+  gc_by_sym     = ID2SYM(rb_intern_const("gc_by"));
+  newobj_sym    = ID2SYM(rb_intern_const("newobj"));
+  malloc_sym    = ID2SYM(rb_intern_const("malloc"));
+  method_sym    = ID2SYM(rb_intern_const("method"));
+  capi_sym      = ID2SYM(rb_intern_const("capi"));
+  stress_sym    = ID2SYM(rb_intern_const("stress"));
+  major_by_sym  = ID2SYM(rb_intern_const("major_by"));
+  nofree_sym    = ID2SYM(rb_intern_const("nofree"));
+  oldgen_sym    = ID2SYM(rb_intern_const("oldgen"));
+  shady_sym     = ID2SYM(rb_intern_const("shady"));
+  force_sym     = ID2SYM(rb_intern_const("force"));
+  oldmalloc_sym = ID2SYM(rb_intern_const("oldmalloc"));
+  state_sym     = ID2SYM(rb_intern_const("state"));
+  none_sym      = ID2SYM(rb_intern_const("none"));
+}
+
+bool gc_profiling_has_major_gc_finished(void) {
+  return rb_gc_latest_gc_info(state_sym) == none_sym && rb_gc_latest_gc_info(major_by_sym) != Qnil;
+}
+
+uint8_t gc_profiling_set_metadata(ddog_prof_Label *labels, int labels_length) {
+  uint8_t max_label_count =
+    1 + // thread id
+    1 + // thread name
+    1 + // state
+    1 + // event
+    1 + // gc reason
+    1 + // gc cause
+    1;  // gc type
+
+  if (max_label_count > labels_length) {
+    rb_raise(rb_eArgError, "BUG: gc_profiling_set_metadata invalid labels_length (%d) < max_label_count (%d)", labels_length, max_label_count);
+  }
+
+  uint8_t label_pos = 0;
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("thread id"),
+    .str = DDOG_CHARSLICE_C("GC"),
+    .num = 0, // This shouldn't be needed but the tracer-2.7 docker image ships a buggy gcc that complains about this
+  };
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("thread name"),
+    .str = DDOG_CHARSLICE_C("Garbage Collection"),
+    .num = 0, // Workaround, same as above
+  };
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("state"),
+    .str = DDOG_CHARSLICE_C("had cpu"),
+    .num = 0, // Workaround, same as above
+  };
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("event"),
+    .str = DDOG_CHARSLICE_C("gc"),
+    .num = 0, // Workaround, same as above
+  };
+
+  VALUE major_by = rb_gc_latest_gc_info(major_by_sym);
+  if (major_by != Qnil) {
+    labels[label_pos++] = (ddog_prof_Label) {
+      .key = DDOG_CHARSLICE_C("gc reason"),
+      .str = major_gc_reason_pretty(major_by),
+    };
+  }
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("gc cause"),
+    .str = gc_cause_pretty(rb_gc_latest_gc_info(gc_by_sym)),
+  };
+
+  labels[label_pos++] = (ddog_prof_Label) {
+    .key = DDOG_CHARSLICE_C("gc type"),
+    .str = gc_type_pretty(major_by, rb_gc_latest_gc_info(state_sym)),
+  };
+
+  if (label_pos > max_label_count) {
+    rb_raise(rb_eRuntimeError, "BUG: gc_profiling_set_metadata unexpected label_pos (%d) > max_label_count (%d)", label_pos, max_label_count);
+  }
+
+  return label_pos;
+}
+
+static ddog_CharSlice major_gc_reason_pretty(VALUE major_gc_reason) {
+  if (major_gc_reason == nofree_sym   ) return DDOG_CHARSLICE_C("not enough free slots (NOFREE)");
+  if (major_gc_reason == oldgen_sym   ) return DDOG_CHARSLICE_C("old generation full (OLDGEN)");
+  if (major_gc_reason == shady_sym    ) return DDOG_CHARSLICE_C("too many objects without write barriers (SHADY)");
+  if (major_gc_reason == force_sym    ) return DDOG_CHARSLICE_C("requested (FORCE)");
+  if (major_gc_reason == oldmalloc_sym) return DDOG_CHARSLICE_C("heap bytes allocated threshold (OLDMALLOC)");
+  return DDOG_CHARSLICE_C("unknown");
+}
+
+static ddog_CharSlice gc_cause_pretty(VALUE gc_cause) {
+  if (gc_cause == newobj_sym) return DDOG_CHARSLICE_C("object allocation");
+  if (gc_cause == malloc_sym) return DDOG_CHARSLICE_C("malloc()");
+  if (gc_cause == method_sym) return DDOG_CHARSLICE_C("GC.start()");
+  if (gc_cause == capi_sym  ) return DDOG_CHARSLICE_C("rb_gc()");
+  if (gc_cause == stress_sym) return DDOG_CHARSLICE_C("stress");
+  return DDOG_CHARSLICE_C("unknown");
+}
+
+static ddog_CharSlice gc_type_pretty(VALUE major_gc_reason, VALUE gc_state) {
+  if (major_gc_reason != Qnil) {
+    if (gc_state == marking_sym ) return DDOG_CHARSLICE_C("major (ongoing, marking)");
+    if (gc_state == sweeping_sym) return DDOG_CHARSLICE_C("major (ongoing, sweeping)");
+    return DDOG_CHARSLICE_C("major");
+  } else {
+    // As we delay flushing events when a minor GC finishes, it's not relevant to include the observed state of the
+    // minor GC, as we often won't record a marking -> sweeping -> done cycle, as it happens too quickly.
+    return DDOG_CHARSLICE_C("minor");
+  }
+}

data/ext/datadog_profiling_native_extension/collectors_gc_profiling_helper.h

@@ -0,0 +1,5 @@
+#pragma once
+
+void gc_profiling_init(void);
+bool gc_profiling_has_major_gc_finished(void);
+uint8_t gc_profiling_set_metadata(ddog_prof_Label *labels, int labels_length);

data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.c

@@ -11,9 +11,6 @@
 // Gathers stack traces from running threads, storing them in a StackRecorder instance
 // This file implements the native bits of the Datadog::Profiling::Collectors::Stack class
 
-#define MAX_FRAMES_LIMIT            10000
-#define MAX_FRAMES_LIMIT_AS_STRING "10000"
-
 static VALUE missing_string = Qnil;
 
 // Used as scratch space during sampling
@@ -36,23 +33,7 @@ static VALUE _native_sample(
   VALUE in_gc
 );
 static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer* buffer, char *frames_omitted_message, int frames_omitted_message_size);
-static void record_placeholder_stack_in_native_code(
-  sampling_buffer* buffer,
-  VALUE recorder_instance,
-  sample_values values,
-  sample_labels labels,
-  sampling_buffer *record_buffer,
-  int extra_frames_in_record_buffer
-);
-static void sample_thread_internal(
-  VALUE thread,
-  sampling_buffer* buffer,
-  VALUE recorder_instance,
-  sample_values values,
-  sample_labels labels,
-  sampling_buffer *record_buffer,
-  int extra_frames_in_record_buffer
-);
+static void record_placeholder_stack_in_native_code(sampling_buffer* buffer, VALUE recorder_instance, sample_values values, sample_labels labels);
 
 void collectors_stack_init(VALUE profiling_module) {
   VALUE collectors_module = rb_define_module_under(profiling_module, "Collectors");
@@ -88,6 +69,7 @@ static VALUE _native_sample(
     .cpu_or_wall_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("cpu-samples"), zero)),
     .wall_time_ns  = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("wall-time"),     zero)),
     .alloc_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("alloc-samples"), zero)),
+    .timeline_wall_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("timeline"), zero)),
   };
 
   long labels_count = RARRAY_LEN(labels_array) + RARRAY_LEN(numeric_labels_array);
@@ -122,62 +104,29 @@ static VALUE _native_sample(
 
   ddog_prof_Slice_Label slice_labels = {.ptr = labels, .len = labels_count};
 
-  sample_thread(
-    thread,
-    buffer,
-    recorder_instance,
-    values,
-    (sample_labels) {.labels = slice_labels, .state_label = state_label},
-    RTEST(in_gc) ? SAMPLE_IN_GC : SAMPLE_REGULAR
-  );
+  if (in_gc == Qtrue) {
+    record_placeholder_stack(
+      buffer,
+      recorder_instance,
+      values,
+      (sample_labels) {.labels = slice_labels, .state_label = state_label},
+      DDOG_CHARSLICE_C("Garbage Collection")
+    );
+  } else {
+    sample_thread(
+      thread,
+      buffer,
+      recorder_instance,
+      values,
+      (sample_labels) {.labels = slice_labels, .state_label = state_label}
+    );
+  }
 
   sampling_buffer_free(buffer);
 
   return Qtrue;
 }
 
-void sample_thread(
-  VALUE thread,
-  sampling_buffer* buffer,
-  VALUE recorder_instance,
-  sample_values values,
-  sample_labels labels,
-  sample_type type
-) {
-  // Samples thread into recorder
-  if (type == SAMPLE_REGULAR) {
-    sampling_buffer *record_buffer = buffer;
-    int extra_frames_in_record_buffer = 0;
-    sample_thread_internal(thread, buffer, recorder_instance, values, labels, record_buffer, extra_frames_in_record_buffer);
-    return;
-  }
-
-  // Samples thread into recorder, including as a top frame in the stack a frame named "Garbage Collection"
-  if (type == SAMPLE_IN_GC) {
-    ddog_CharSlice function_name = DDOG_CHARSLICE_C("");
-    ddog_CharSlice function_filename = DDOG_CHARSLICE_C("Garbage Collection");
-    buffer->locations[0] = (ddog_prof_Location) {
-      .function = (ddog_prof_Function) {.name = function_name, .filename = function_filename},
-      .line = 0
-    };
-    // To avoid changing sample_thread_internal, we just prepare a new buffer struct that uses the same underlying storage as the
-    // original buffer, but has capacity one less, so that we can keep the above Garbage Collection frame untouched.
-    sampling_buffer thread_in_gc_buffer = (struct sampling_buffer) {
-      .max_frames = buffer->max_frames - 1,
-      .stack_buffer = buffer->stack_buffer + 1,
-      .lines_buffer = buffer->lines_buffer + 1,
-      .is_ruby_frame = buffer->is_ruby_frame + 1,
-      .locations = buffer->locations + 1,
-    };
-    sampling_buffer *record_buffer = buffer; // We pass in the original buffer as the record_buffer, but not as the regular buffer
-    int extra_frames_in_record_buffer = 1;
-    sample_thread_internal(thread, &thread_in_gc_buffer, recorder_instance, values, labels, record_buffer, extra_frames_in_record_buffer);
-    return;
-  }
-
-  rb_raise(rb_eArgError, "Unexpected value for sample_type: %d", type);
-}
-
 #define CHARSLICE_EQUALS(must_be_a_literal, charslice) (strlen("" must_be_a_literal) == charslice.len && strncmp(must_be_a_literal, charslice.ptr, charslice.len) == 0)
 
 // Idea: Should we release the global vm lock (GVL) after we get the data from `rb_profile_frames`? That way other Ruby threads
@@ -189,24 +138,12 @@ void sample_thread(
 // * Should we move this into a different thread entirely?
 // * If we don't move it into a different thread, does releasing the GVL on a Ruby thread mean that we're introducing
 //   a new thread switch point where there previously was none?
-//
-// ---
-//
-// Why the weird extra record_buffer and extra_frames_in_record_buffer?
-// The answer is: to support both sample_thread() and sample_thread_in_gc().
-//
-// For sample_thread(), buffer == record_buffer and extra_frames_in_record_buffer == 0, so it's a no-op.
-// For sample_thread_in_gc(), the buffer is a special buffer that is the same as the record_buffer, but with every
-// pointer shifted forward extra_frames_in_record_buffer elements, so that the caller can actually inject those extra
-// frames, and this function doesn't have to care about it.
-static void sample_thread_internal(
+void sample_thread(
   VALUE thread,
   sampling_buffer* buffer,
   VALUE recorder_instance,
   sample_values values,
-  sample_labels labels,
-  sampling_buffer *record_buffer,
-  int extra_frames_in_record_buffer
+  sample_labels labels
 ) {
   int captured_frames = ddtrace_rb_profile_frames(
     thread,
@@ -218,14 +155,7 @@ static void sample_thread_internal(
   );
 
   if (captured_frames == PLACEHOLDER_STACK_IN_NATIVE_CODE) {
-    record_placeholder_stack_in_native_code(
-      buffer,
-      recorder_instance,
-      values,
-      labels,
-      record_buffer,
-      extra_frames_in_record_buffer
-    );
+    record_placeholder_stack_in_native_code(buffer, recorder_instance, values, labels);
     return;
   }
 
@@ -332,7 +262,7 @@ static void sample_thread_internal(
 
   record_sample(
     recorder_instance,
-    (ddog_prof_Slice_Location) {.ptr = record_buffer->locations, .len = captured_frames + extra_frames_in_record_buffer},
+    (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = captured_frames},
     values,
     labels
   );
@@ -379,24 +309,35 @@ static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer*
 //
 // To give customers visibility into these threads, rather than reporting an empty stack, we replace the empty stack
 // with one containing a placeholder frame, so that these threads are properly represented in the UX.
+
 static void record_placeholder_stack_in_native_code(
+  sampling_buffer* buffer,
+  VALUE recorder_instance,
+  sample_values values,
+  sample_labels labels
+) {
+  record_placeholder_stack(
+    buffer,
+    recorder_instance,
+    values,
+    labels,
+    DDOG_CHARSLICE_C("In native code")
+  );
+}
+
+void record_placeholder_stack(
   sampling_buffer* buffer,
   VALUE recorder_instance,
   sample_values values,
   sample_labels labels,
-  sampling_buffer *record_buffer,
-  int extra_frames_in_record_buffer
+  ddog_CharSlice placeholder_stack
 ) {
-  ddog_CharSlice function_name = DDOG_CHARSLICE_C("");
-  ddog_CharSlice function_filename = DDOG_CHARSLICE_C("In native code");
-  buffer->locations[0] = (ddog_prof_Location) {
-    .function = (ddog_prof_Function) {.name = function_name, .filename = function_filename},
-    .line = 0
-  };
+  ddog_prof_Function placeholder = {.name = DDOG_CHARSLICE_C(""), .filename = placeholder_stack};
+  buffer->locations[0] = (ddog_prof_Location) {.function = placeholder, .line = 0};
 
   record_sample(
     recorder_instance,
-    (ddog_prof_Slice_Location) {.ptr = record_buffer->locations, .len = 1 + extra_frames_in_record_buffer},
+    (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = 1},
     values,
     labels
   );

data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.h

@@ -4,17 +4,24 @@
 
 #include "stack_recorder.h"
 
-typedef struct sampling_buffer sampling_buffer;
+#define MAX_FRAMES_LIMIT            10000
+#define MAX_FRAMES_LIMIT_AS_STRING "10000"
 
-typedef enum { SAMPLE_REGULAR, SAMPLE_IN_GC } sample_type;
+typedef struct sampling_buffer sampling_buffer;
 
 void sample_thread(
   VALUE thread,
+  sampling_buffer* buffer,
+  VALUE recorder_instance,
+  sample_values values,
+  sample_labels labels
+);
+void record_placeholder_stack(
   sampling_buffer* buffer,
   VALUE recorder_instance,
   sample_values values,
   sample_labels labels,
-  sample_type type
+  ddog_CharSlice placeholder_stack
 );
 sampling_buffer *sampling_buffer_new(unsigned int max_frames);
 void sampling_buffer_free(sampling_buffer *buffer);