ddtrace 0.54.2 → 1.2.0

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h

@@ -1,3 +1,36 @@
 #pragma once
 
+#include <stdbool.h>
+
+// The private_vm_api_access.c includes the RUBY_MJIT_HEADER which replaces and conflicts with any other Ruby headers;
+// so we use PRIVATE_VM_API_ACCESS_SKIP_RUBY_INCLUDES to be able to include private_vm_api_access.h on that file
+// without also dragging the incompatible includes
+#ifndef PRIVATE_VM_API_ACCESS_SKIP_RUBY_INCLUDES
+  #ifdef RUBY_2_1_WORKAROUND
+    #include <thread_native.h>
+  #else
+    #include <ruby/thread_native.h>
+  #endif
+#endif
+
+#include "extconf.h"
+
 rb_nativethread_id_t pthread_id_for(VALUE thread);
+ptrdiff_t stack_depth_for(VALUE thread);
+VALUE ddtrace_thread_list(void);
+bool is_thread_alive(VALUE thread);
+
+int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, VALUE *buff, int *lines, bool* is_ruby_frame);
+
+// Ruby 3.0 finally added support for showing CFUNC frames (frames for methods written using native code)
+// in stack traces gathered via `rb_profile_frames` (https://github.com/ruby/ruby/pull/3299).
+// To access this information on older Rubies, beyond using our custom `ddtrace_rb_profile_frames` above, we also need
+// to backport the Ruby 3.0+ version of `rb_profile_frame_method_name`.
+#ifdef USE_BACKPORTED_RB_PROFILE_FRAME_METHOD_NAME
+  VALUE ddtrace_rb_profile_frame_method_name(VALUE frame);
+#else // Ruby > 3.0, just use the stock functionality
+  #define ddtrace_rb_profile_frame_method_name rb_profile_frame_method_name
+#endif
+
+// See comment on `record_placeholder_stack_in_native_code` for a full explanation of what this means (and why we don't just return 0)
+#define PLACEHOLDER_STACK_IN_NATIVE_CODE -1

data/ext/ddtrace_profiling_native_extension/profiling.c

@@ -2,9 +2,17 @@
 
 #include "clock_id.h"
 
+// Each class/module here is implemented in their separate file
+void collectors_cpu_and_wall_time_init(VALUE profiling_module);
+void collectors_stack_init(VALUE profiling_module);
+void http_transport_init(VALUE profiling_module);
+void stack_recorder_init(VALUE profiling_module);
+
 static VALUE native_working_p(VALUE self);
 
-void Init_ddtrace_profiling_native_extension(void) {
+#define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
+
+void DDTRACE_EXPORT Init_ddtrace_profiling_native_extension(void) {
   VALUE datadog_module = rb_define_module("Datadog");
   VALUE profiling_module = rb_define_module_under(datadog_module, "Profiling");
   VALUE native_extension_module = rb_define_module_under(profiling_module, "NativeExtension");
@@ -13,6 +21,11 @@ void Init_ddtrace_profiling_native_extension(void) {
   rb_funcall(native_extension_module, rb_intern("private_class_method"), 1, ID2SYM(rb_intern("native_working?")));
 
   rb_define_singleton_method(native_extension_module, "clock_id_for", clock_id_for, 1); // from clock_id.h
+
+  collectors_cpu_and_wall_time_init(profiling_module);
+  collectors_stack_init(profiling_module);
+  http_transport_init(profiling_module);
+  stack_recorder_init(profiling_module);
 }
 
 static VALUE native_working_p(VALUE self) {

data/ext/ddtrace_profiling_native_extension/ruby_helpers.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <ruby.h>
+
+// Processes any pending interruptions, including exceptions to be raised.
+// If there's an exception to be raised, it raises it. In that case, this function does not return.
+static inline VALUE process_pending_interruptions(VALUE _unused) {
+  rb_thread_check_ints();
+  return Qnil;
+}
+
+// Calls process_pending_interruptions BUT "rescues" any exceptions to be raised, returning them instead as
+// a non-zero `pending_exception`.
+//
+// Thus, if there's a non-zero `pending_exception`, the caller MUST call `rb_jump_tag(pending_exception)` after any
+// needed clean-ups.
+//
+// Usage example:
+//
+// ```c
+// foo = ruby_xcalloc(...);
+// pending_exception = check_if_pending_exception();
+// if (pending_exception) {
+//   ruby_xfree(foo);
+//   rb_jump_tag(pending_exception); // Re-raises exception
+// }
+// ```
+__attribute__((warn_unused_result))
+static inline int check_if_pending_exception(void) {
+  int pending_exception;
+  rb_protect(process_pending_interruptions, Qnil, &pending_exception);
+  return pending_exception;
+}

data/ext/ddtrace_profiling_native_extension/stack_recorder.c

@@ -0,0 +1,147 @@
+#include <ruby.h>
+#include <ruby/thread.h>
+#include "stack_recorder.h"
+#include "libddprof_helpers.h"
+#include "ruby_helpers.h"
+
+// Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
+// This file implements the native bits of the Datadog::Profiling::StackRecorder class
+
+static VALUE ok_symbol = Qnil; // :ok in Ruby
+static VALUE error_symbol = Qnil; // :error in Ruby
+
+static ID ruby_time_from_id; // id of :ruby_time_from in Ruby
+
+static VALUE stack_recorder_class = Qnil;
+
+struct call_serialize_without_gvl_arguments {
+  ddprof_ffi_Profile *profile;
+  ddprof_ffi_SerializeResult result;
+  bool serialize_ran;
+};
+
+static VALUE _native_new(VALUE klass);
+static void stack_recorder_typed_data_free(void *data);
+static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
+static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time);
+static void *call_serialize_without_gvl(void *call_args);
+
+void stack_recorder_init(VALUE profiling_module) {
+  stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
+
+  // Instances of the StackRecorder class are "TypedData" objects.
+  // "TypedData" objects are special objects in the Ruby VM that can wrap C structs.
+  // In our case, we're going to keep a libddprof profile reference inside our object.
+  //
+  // Because Ruby doesn't know how to initialize libddprof profiles, we MUST override the allocation function for objects
+  // of this class so that we can manage this part. Not overriding or disabling the allocation function is a common
+  // gotcha for "TypedData" objects that can very easily lead to VM crashes, see for instance
+  // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
+  rb_define_alloc_func(stack_recorder_class, _native_new);
+
+  rb_define_singleton_method(stack_recorder_class, "_native_serialize",  _native_serialize, 1);
+
+  ok_symbol = ID2SYM(rb_intern_const("ok"));
+  error_symbol = ID2SYM(rb_intern_const("error"));
+  ruby_time_from_id = rb_intern_const("ruby_time_from");
+}
+
+// This structure is used to define a Ruby object that stores a pointer to a ddprof_ffi_Profile instance
+// See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
+static const rb_data_type_t stack_recorder_typed_data = {
+  .wrap_struct_name = "Datadog::Profiling::StackRecorder",
+  .function = {
+    .dfree = stack_recorder_typed_data_free,
+    .dsize = NULL, // We don't track profile memory usage (although it'd be cool if we did!)
+    // No need to provide dmark nor dcompact because we don't directly reference Ruby VALUEs from inside this object
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE _native_new(VALUE klass) {
+  ddprof_ffi_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
+
+  ddprof_ffi_Profile *profile = ddprof_ffi_Profile_new(sample_types, NULL /* Period is optional */);
+
+  return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, profile);
+}
+
+static void stack_recorder_typed_data_free(void *data) {
+  ddprof_ffi_Profile_free((ddprof_ffi_Profile *) data);
+}
+
+static VALUE _native_serialize(VALUE self, VALUE recorder_instance) {
+  ddprof_ffi_Profile *profile;
+  TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
+
+  // We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
+  // is pending
+  struct call_serialize_without_gvl_arguments args = {.profile = profile, .serialize_ran = false};
+
+  while (!args.serialize_ran) {
+    // Give the Ruby VM an opportunity to process any pending interruptions (including raising exceptions).
+    // Note that it's OK to do this BEFORE call_serialize_without_gvl runs BUT NOT AFTER because afterwards
+    // there's heap-allocated memory that MUST be cleaned before raising any exception.
+    //
+    // Note that we run this in a loop because `rb_thread_call_without_gvl2` may return multiple times due to
+    // pending interrupts until it actually runs our code.
+    process_pending_interruptions(Qnil);
+
+    // We use rb_thread_call_without_gvl2 here because unlike the regular _gvl variant, gvl2 does not process
+    // interruptions and thus does not raise exceptions after running our code.
+    rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, /* No interruption function supported */ NULL, NULL);
+  }
+
+  ddprof_ffi_SerializeResult serialized_profile = args.result;
+
+  if (serialized_profile.tag == DDPROF_FFI_SERIALIZE_RESULT_ERR) {
+    VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
+    ddprof_ffi_SerializeResult_drop(serialized_profile);
+    return rb_ary_new_from_args(2, error_symbol, err_details);
+  }
+
+  VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
+
+  ddprof_ffi_Timespec ddprof_start = serialized_profile.ok.start;
+  ddprof_ffi_Timespec ddprof_finish = serialized_profile.ok.end;
+
+  // Clean up libddprof object to avoid leaking in case ruby_time_from raises an exception
+  ddprof_ffi_SerializeResult_drop(serialized_profile);
+
+  VALUE start = ruby_time_from(ddprof_start);
+  VALUE finish = ruby_time_from(ddprof_finish);
+
+  if (!ddprof_ffi_Profile_reset(profile)) return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
+
+  return rb_ary_new_from_args(2, ok_symbol, rb_ary_new_from_args(3, start, finish, encoded_pprof));
+}
+
+static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time) {
+  #ifndef NO_RB_TIME_TIMESPEC_NEW // Modern Rubies
+    const int utc = INT_MAX - 1; // From Ruby sources
+    struct timespec time = {.tv_sec = ddprof_time.seconds, .tv_nsec = ddprof_time.nanoseconds};
+    return rb_time_timespec_new(&time, utc);
+  #else // Ruby < 2.3
+    return rb_funcall(stack_recorder_class, ruby_time_from_id, 2, LONG2NUM(ddprof_time.seconds), UINT2NUM(ddprof_time.nanoseconds));
+  #endif
+}
+
+void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample) {
+  ddprof_ffi_Profile *profile;
+  TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
+
+  ddprof_ffi_Profile_add(profile, sample);
+}
+
+static void *call_serialize_without_gvl(void *call_args) {
+  struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
+
+  args->result = ddprof_ffi_Profile_serialize(args->profile);
+  args->serialize_ran = true;
+
+  return NULL; // Unused
+}
+
+void enforce_recorder_instance(VALUE object) {
+  Check_TypedStruct(object, &stack_recorder_typed_data);
+}

data/ext/ddtrace_profiling_native_extension/stack_recorder.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <ddprof/ffi.h>
+
+// Note: Please DO NOT use `VALUE_STRING` anywhere else, instead use `DDPROF_FFI_CHARSLICE_C`.
+// `VALUE_STRING` is only needed because older versions of gcc (4.9.2, used in our Ruby 2.1 and 2.2 CI test images)
+// tripped when compiling `enabled_value_types` using `-std=gnu99` due to the extra cast that is included in
+// `DDPROF_FFI_CHARSLICE_C` with the following error:
+//
+// ```
+// compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
+// ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
+// static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
+// ^
+// ```
+#define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
+
+#define      CPU_TIME_VALUE {.type_ = VALUE_STRING("cpu-time"),      .unit = VALUE_STRING("nanoseconds")}
+#define   CPU_SAMPLES_VALUE {.type_ = VALUE_STRING("cpu-samples"),   .unit = VALUE_STRING("count")}
+#define     WALL_TIME_VALUE {.type_ = VALUE_STRING("wall-time"),     .unit = VALUE_STRING("nanoseconds")}
+#define ALLOC_SAMPLES_VALUE {.type_ = VALUE_STRING("alloc-samples"), .unit = VALUE_STRING("count")}
+#define   ALLOC_SPACE_VALUE {.type_ = VALUE_STRING("alloc-space"),   .unit = VALUE_STRING("bytes")}
+#define    HEAP_SPACE_VALUE {.type_ = VALUE_STRING("heap-space"),    .unit = VALUE_STRING("bytes")}
+
+static const ddprof_ffi_ValueType enabled_value_types[] = {
+  #define CPU_TIME_VALUE_POS 0
+  CPU_TIME_VALUE,
+  #define CPU_SAMPLES_VALUE_POS 1
+  CPU_SAMPLES_VALUE,
+  #define WALL_TIME_VALUE_POS 2
+  WALL_TIME_VALUE
+};
+
+#define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddprof_ffi_ValueType))
+
+void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample);
+void enforce_recorder_instance(VALUE object);