datadog 2.20.0 → 2.26.0

data/ext/datadog_profiling_native_extension/crashtracking_runtime_stacks.c

@@ -0,0 +1,239 @@
+// NOTE: This file is a part of the profiling native extension even though the
+// runtime stacks feature is consumed by the crashtracker. The profiling
+// extension already carries all the Ruby VM private header access and build
+// plumbing required to safely poke at internal structures. Sharing that setup
+// avoids duplicating another native extension with the same (fragile) access
+// patterns, and keeps the overall install/build surface area smaller.
+//
+// This also means that this functionality is depend on the profiling native extension
+// being available and built
+#include "extconf.h"
+
+#if defined(__linux__)
+
+#include <datadog/crashtracker.h>
+#include "datadog_ruby_common.h"
+#include "private_vm_api_access.h"
+#include <sys/mman.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+
+static const rb_data_type_t *crashtracker_thread_data_type = NULL;
+
+static void ruby_runtime_stack_callback(
+  void (*emit_frame)(const ddog_crasht_RuntimeStackFrame*)
+);
+
+// Use a fixed, preallocated buffer for crash-time runtime stacks to avoid
+// heap allocation in the signal/crash path.
+#define RUNTIME_STACK_MAX_FRAMES 512
+static frame_info runtime_stack_buffer[RUNTIME_STACK_MAX_FRAMES];
+
+#if defined(__x86_64__)
+#  define SYS_MINCORE 0x1B
+#elif defined(__aarch64__)
+#  define SYS_MINCORE 0xE8
+#endif
+
+long syscall(long number, ...);
+
+// align down to power of two
+static inline uintptr_t align_down(uintptr_t x, uintptr_t align) {
+  return x & ~(align - 1u);
+}
+
+static uintptr_t cached_page_size = 0;
+
+// TODO: This function is not necessarily Ruby specific. This will be moved to
+// `libdatadog` in the future as a shared utility function.
+static inline bool is_pointer_readable(const void *ptr, size_t size) {
+  if (!ptr || size == 0) return false;
+
+  uintptr_t page_size = cached_page_size;
+
+  const uintptr_t start = align_down((uintptr_t)ptr, page_size);
+  const uintptr_t end   = ((uintptr_t)ptr + size - 1u);
+  const uintptr_t last  = align_down(end, page_size);
+
+  // Number of pages spanned
+  size_t pages = 1u + (last != start);
+  if (pages > 2u) pages = 2u;
+
+  unsigned char vec[2];
+
+  int retries = 5;
+  for (;;) {
+      size_t len = pages * (size_t)page_size;
+      long rc = syscall(SYS_MINCORE, (void*)start, len, vec);
+
+      if (rc == 0) {
+          return true;
+      }
+
+      int e = errno;
+      if (e == ENOMEM || e == EFAULT) {
+          return false;
+      }
+
+      if (e == EAGAIN && retries-- > 0) {
+          continue;
+      }
+
+      // Unknown errno, we assume mapped to avoid cascading faults in crash path
+      return true;
+  }
+}
+
+static inline ddog_CharSlice char_slice_from_cstr(const char *cstr) {
+  if (cstr == NULL) {
+    return (ddog_CharSlice){.ptr = NULL, .len = 0};
+  }
+  return (ddog_CharSlice){.ptr = cstr, .len = strlen(cstr)};
+}
+
+static ddog_CharSlice safe_string_value(VALUE str) {
+  if (str == Qnil) return DDOG_CHARSLICE_C("<nil>");
+
+  // Validate object and payload readability in one check
+  if (!is_pointer_readable((const void *)str, sizeof(struct RString))) return DDOG_CHARSLICE_C("<corrupted>");
+  if (!RB_TYPE_P(str, T_STRING)) return DDOG_CHARSLICE_C("<not_string>");
+
+  long len = RSTRING_LEN(str);
+  if (len < 0 || len > 1024) return DDOG_CHARSLICE_C("<length_over_limit>");
+
+  const char *ptr = RSTRING_PTR(str);
+  if (!ptr) return DDOG_CHARSLICE_C("<null>");
+
+  if (!is_pointer_readable(ptr, len > 0 ? len : 1)) return DDOG_CHARSLICE_C("<unreadable>");
+
+  return (ddog_CharSlice){.ptr = ptr, .len = (size_t)len};
+}
+
+static void emit_placeholder_frame(
+  void (*emit_frame)(const ddog_crasht_RuntimeStackFrame*),
+  const char *label
+) {
+  ddog_crasht_RuntimeStackFrame frame = {
+    .type_name = DDOG_CHARSLICE_C(""),
+    .function = char_slice_from_cstr(label),
+    .file = DDOG_CHARSLICE_C("<unknown>"),
+    .line = 0,
+    .column = 0
+  };
+  emit_frame(&frame);
+}
+
+// Collect the crashing thread's frames via ddtrace_rb_profile_frames into a static buffer, then emit
+// them newest-first. If corruption is detected, emit placeholder frames so the crash report still
+// completes. We lean on the Ruby VM helpers we already use for profiling and rely on crashtracker's
+// safety nets so a failure here should not impact customers.
+static void ruby_runtime_stack_callback(
+  void (*emit_frame)(const ddog_crasht_RuntimeStackFrame*)
+) {
+  // Grab the Ruby thread we crashed on; crashtracker only runs once.
+  VALUE current_thread = rb_thread_current();
+  if (current_thread == Qnil) return;
+
+  if (crashtracker_thread_data_type == NULL) return;
+
+  if (!rb_typeddata_is_kind_of(current_thread, crashtracker_thread_data_type)) {
+    emit_placeholder_frame(emit_frame, "<runtime stack not found>");
+    return;
+  }
+
+  // Use the profiling helper to gather frames into our static buffer.
+  int frame_count = ddtrace_rb_profile_frames(
+    current_thread,
+    0,
+    RUNTIME_STACK_MAX_FRAMES,
+    runtime_stack_buffer
+  );
+
+  if (frame_count <= 0) {
+    emit_placeholder_frame(emit_frame, "<runtime stack not found>");
+    return;
+  }
+
+  for (int i = frame_count - 1; i >= 0; i--) {
+    frame_info *info = &runtime_stack_buffer[i];
+
+    if (info->is_ruby_frame) {
+      const void *iseq = (const void *)info->as.ruby_frame.iseq;
+      ddog_CharSlice function_slice = DDOG_CHARSLICE_C("<unknown>");
+      ddog_CharSlice file_slice = DDOG_CHARSLICE_C("<unknown>");
+
+      if (iseq && is_pointer_readable(iseq, sizeof_rb_iseq_t())) {
+        function_slice = safe_string_value(ddtrace_iseq_base_label(iseq));
+        file_slice = safe_string_value(ddtrace_iseq_path(iseq));
+      }
+
+      ddog_crasht_RuntimeStackFrame frame = {
+        .type_name = DDOG_CHARSLICE_C(""),
+        .function = function_slice,
+        .file = file_slice,
+        .line = info->as.ruby_frame.line,
+        .column = 0
+      };
+
+      emit_frame(&frame);
+    } else {
+      ddog_CharSlice function_slice = DDOG_CHARSLICE_C("<C method>");
+      ddog_CharSlice file_slice = DDOG_CHARSLICE_C("<C extension>");
+
+      if (info->as.native_frame.method_id) {
+        const char *method_name = rb_id2name(info->as.native_frame.method_id);
+        if (is_pointer_readable(method_name, 256)) {
+          size_t method_name_len = strnlen(method_name, 256);
+          if (method_name_len > 0 && method_name_len < 256) {
+            function_slice = char_slice_from_cstr(method_name);
+          }
+        }
+      }
+
+      ddog_crasht_RuntimeStackFrame frame = {
+        .type_name = DDOG_CHARSLICE_C(""),
+        .function = function_slice,
+        .file = file_slice,
+        .line = 0,
+        .column = 0
+      };
+
+      emit_frame(&frame);
+    }
+  }
+
+  if (frame_count == RUNTIME_STACK_MAX_FRAMES) {
+    emit_placeholder_frame(emit_frame, "<truncated frames>");
+  }
+}
+
+void crashtracking_runtime_stacks_init(void) {
+  cached_page_size = (uintptr_t)sysconf(_SC_PAGESIZE);
+  if (cached_page_size == 0 || (cached_page_size & (cached_page_size - 1u))) {
+    cached_page_size = 4096;
+  }
+
+  if (crashtracker_thread_data_type == NULL) {
+    VALUE current_thread = rb_thread_current();
+    if (current_thread == Qnil) {
+      rb_raise(rb_eRuntimeError, "crashtracking_runtime_stacks_init: rb_thread_current returned Qnil");
+    }
+
+    const rb_data_type_t *thread_data_type = RTYPEDDATA_TYPE(current_thread);
+    if (!thread_data_type) {
+      rb_raise(rb_eRuntimeError, "crashtracking_runtime_stacks_init: RTYPEDDATA_TYPE returned NULL");
+    }
+
+    crashtracker_thread_data_type = thread_data_type;
+  }
+
+  // Register immediately so Ruby doesn't need to manage this explicitly.
+  ddog_crasht_register_runtime_frame_callback(ruby_runtime_stack_callback);
+}
+
+#else
+// Keep init symbol to satisfy linkage on non linux platforms, but do nothing
+void crashtracking_runtime_stacks_init(void) {}
+#endif
+

data/ext/datadog_profiling_native_extension/datadog_ruby_common.h

@@ -3,7 +3,7 @@
 // IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
 
 #include <ruby.h>
-#include <datadog/profiling.h>
+#include <datadog/common.h>
 
 // Used to mark symbols to be exported to the outside of the extension.
 // Consider very carefully before tagging a function with this.

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -138,11 +138,16 @@ if have_header("dlfcn.h")
     have_func("dladdr")
 end
 
-# On Ruby 3.5, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
-$defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "3.5"
+# On older Rubies, there was no primitive mutex and condition variable implemented in `thread_sync.rb` (internal)
+$defs << "-DNO_PRIMITIVE_MUTEX_AND_CONDITION_VARIABLE" if RUBY_VERSION < "4"
 
-# On Ruby 2.5 and 3.3, this symbol was not visible. It is on 2.6 to 3.2, as well as 3.4+
-$defs << "-DNO_RB_OBJ_INFO" if RUBY_VERSION.start_with?("2.5", "3.3")
+# On Ruby 4, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
+$defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
+
+# This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
+# It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
+# actually required for correct behavior of the profiler.
+$defs << "-DNO_RB_OBJ_INFO" if RUBY_VERSION.start_with?("2.5", "3.3", "4.")
 
 # On older Rubies, rb_postponed_job_preregister/rb_postponed_job_trigger did not exist
 $defs << "-DNO_POSTPONED_TRIGGER" if RUBY_VERSION < "3.3"

data/ext/datadog_profiling_native_extension/heap_recorder.c

@@ -305,7 +305,7 @@ void start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj
 
   if (++heap_recorder->num_recordings_skipped < heap_recorder->sample_rate ||
       #ifdef NO_IMEMO_OBJECT_ID
-        // On Ruby 3.5, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
+        // On Ruby 4, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
         RB_BUILTIN_TYPE(new_obj) == RUBY_T_IMEMO
       #else
         false

data/ext/datadog_profiling_native_extension/http_transport.c

@@ -136,6 +136,7 @@ static VALUE perform_export(
     files_to_compress_and_export,
     /* files_to_export_unmodified: */ ddog_prof_Exporter_Slice_File_empty(),
     /* optional_additional_tags: */ NULL,
+    /* optional_process_tags: */ NULL,
     &internal_metadata,
     &info
   );

data/ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -76,6 +76,18 @@ rb_nativethread_id_t pthread_id_for(VALUE thread) {
   #endif
 }
 
+size_t sizeof_rb_iseq_t(void) {
+  return sizeof(rb_iseq_t);
+}
+
+VALUE ddtrace_iseq_base_label(const void *iseq) {
+  return rb_iseq_base_label((const rb_iseq_t *)iseq);
+}
+
+VALUE ddtrace_iseq_path(const void *iseq) {
+  return rb_iseq_path((const rb_iseq_t *)iseq);
+}
+
 // Queries if the current thread is the owner of the global VM lock.
 //
 // @ivoanjo: Ruby has a similarly-named `ruby_thread_has_gvl_p` but that API is insufficient for our needs because it can

data/ext/datadog_profiling_native_extension/private_vm_api_access.h

@@ -47,6 +47,10 @@ VALUE thread_name_for(VALUE thread);
 
 int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, frame_info *stack_buffer);
 
+size_t sizeof_rb_iseq_t(void);
+VALUE ddtrace_iseq_base_label(const void *iseq);
+VALUE ddtrace_iseq_path(const void *iseq);
+
 // Returns true if the current thread belongs to the main Ractor or if Ruby has no Ractor support
 bool ddtrace_rb_ractor_main_p(void);
 

data/ext/datadog_profiling_native_extension/profiling.c

@@ -23,6 +23,7 @@ void collectors_thread_context_init(VALUE profiling_module);
 void encoded_profile_init(VALUE profiling_module);
 void http_transport_init(VALUE profiling_module);
 void stack_recorder_init(VALUE profiling_module);
+void crashtracking_runtime_stacks_init(void);
 
 static VALUE native_working_p(VALUE self);
 static VALUE _native_grab_gvl_and_raise(DDTRACE_UNUSED VALUE _self, VALUE exception_class, VALUE test_message, VALUE test_message_arg, VALUE release_gvl);
@@ -65,6 +66,7 @@ void DDTRACE_EXPORT Init_datadog_profiling_native_extension(void) {
   encoded_profile_init(profiling_module);
   http_transport_init(profiling_module);
   stack_recorder_init(profiling_module);
+  crashtracking_runtime_stacks_init();
   unsafe_api_calls_check_init();
 
   // Hosts methods used for testing the native code using RSpec

data/ext/libdatadog_api/datadog_ruby_common.h

@@ -3,7 +3,7 @@
 // IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
 
 #include <ruby.h>
-#include <datadog/profiling.h>
+#include <datadog/common.h>
 
 // Used to mark symbols to be exported to the outside of the extension.
 // Consider very carefully before tagging a function with this.

data/ext/libdatadog_api/ddsketch.c

@@ -0,0 +1,106 @@
+#include <ruby.h>
+#include <datadog/ddsketch.h>
+
+#include "datadog_ruby_common.h"
+
+static VALUE _native_new(VALUE klass);
+static void ddsketch_free(void *ptr);
+static VALUE native_add(VALUE self, VALUE point);
+static VALUE native_add_with_count(VALUE self, VALUE point, VALUE count);
+static VALUE native_count(VALUE self);
+static VALUE native_encode(VALUE self);
+NORETURN(static void raise_ddsketch_error(const char *message, ddog_VoidResult result));
+
+void ddsketch_init(VALUE core_module) {
+  VALUE ddsketch_class = rb_define_class_under(core_module, "DDSketch", rb_cObject);
+
+  rb_define_alloc_func(ddsketch_class, _native_new);
+  rb_define_method(ddsketch_class, "add", native_add, 1);
+  rb_define_method(ddsketch_class, "add_with_count", native_add_with_count, 2);
+  rb_define_method(ddsketch_class, "count", native_count, 0);
+  rb_define_method(ddsketch_class, "encode", native_encode, 0);
+}
+
+// This structure is used to define a Ruby object that stores a pointer to a ddsketch_Handle_DDSketch
+// See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
+static const rb_data_type_t ddsketch_typed_data = {
+  .wrap_struct_name = "Datadog::DDSketch",
+  .function = {
+    .dmark = NULL, // We don't store references to Ruby objects so we don't need to mark any of them
+    .dfree = ddsketch_free,
+    .dsize = NULL, // We don't track memory usage (although it'd be cool if we did!)
+    //.dcompact = NULL, // Not needed -- we don't store references to Ruby objects
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE _native_new(VALUE klass) {
+  ddsketch_Handle_DDSketch *state = ruby_xcalloc(1, sizeof(ddsketch_Handle_DDSketch));
+
+  *state = ddog_ddsketch_new();
+
+  return TypedData_Wrap_Struct(klass, &ddsketch_typed_data, state);
+}
+
+static void ddsketch_free(void *ptr) {
+  ddsketch_Handle_DDSketch *state = (ddsketch_Handle_DDSketch *) ptr;
+  ddog_ddsketch_drop(state);
+  ruby_xfree(ptr);
+}
+
+static void raise_ddsketch_error(const char *message, ddog_VoidResult result) {
+  rb_raise(rb_eRuntimeError, "%s: %"PRIsVALUE, message, get_error_details_and_drop(&result.err));
+}
+
+static VALUE native_add(VALUE self, VALUE point) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_VoidResult result = ddog_ddsketch_add(state, NUM2DBL(point));
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch add failed", result);
+
+  return self;
+}
+
+static VALUE native_add_with_count(VALUE self, VALUE point, VALUE count) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_VoidResult result = ddog_ddsketch_add_with_count(state, NUM2DBL(point), NUM2DBL(count));
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch add_with_count failed", result);
+
+  return self;
+}
+
+static VALUE native_count(VALUE self) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  double count_out;
+  ddog_VoidResult result = ddog_ddsketch_count(state, &count_out);
+
+  if (result.tag == DDOG_VOID_RESULT_ERR) raise_ddsketch_error("DDSketch count failed", result);
+
+  return DBL2NUM(count_out);
+}
+
+static VALUE native_encode(VALUE self) {
+  ddsketch_Handle_DDSketch *state;
+  TypedData_Get_Struct(self, ddsketch_Handle_DDSketch, &ddsketch_typed_data, state);
+
+  ddog_Vec_U8 encoded = ddog_ddsketch_encode(state);
+
+  // Copy into a Ruby string
+  VALUE bytes = rb_str_new((const char *) encoded.ptr, encoded.len);
+
+  ddog_Vec_U8_drop(encoded);
+
+  // The sketch is consumed by encode; to make this a bit more user-friendly for
+  // a Ruby API (since we can't "kill" the Ruby object), let's re-initialize it so
+  // it can be used again.
+  *state = ddog_ddsketch_new();
+
+  return bytes;
+}