datadog 2.39.0 → 2.47.0

data/ext/libdatadog_api/extconf.rb

@@ -24,7 +24,7 @@ end
 if ENV["DD_NO_EXTENSION"].to_s.strip.downcase == "true"
   skip_building_extension!("the `DD_NO_EXTENSION` environment variable is/was set to `true` during installation")
 end
-skip_building_extension!("current Ruby VM is not supported") if RUBY_ENGINE != "ruby"
+skip_building_extension!("current Ruby VM is not supported") unless %w[ruby truffleruby].include?(RUBY_ENGINE)
 skip_building_extension!("Microsoft Windows is not supported") if Gem.win_platform?
 
 libdatadog_issue = Datadog::LibdatadogExtconfHelpers.load_libdatadog_or_get_issue
@@ -98,6 +98,11 @@ EXTENSION_NAME = "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}".fr
 
 have_func("rb_iseq_type")
 
+$defs << "-DHAVE_RB_IMEMO_NAME" if RUBY_VERSION >= "3.0"
+$defs << "-DHAVE_SHIFTED_IMEMO_MASK" if RUBY_VERSION >= "4.1"
+$defs << "-DHAVE_RUBY_THREAD_STORAGE_API" if RUBY_VERSION >= "3.3"
+$defs << "-DHAVE_RUBY_THREAD_HAS_GVL_P" if have_func("ruby_thread_has_gvl_p")
+
 create_makefile(EXTENSION_NAME)
 
 # rubocop:enable Style/GlobalVars

data/ext/libdatadog_api/feature_flags.c

@@ -13,6 +13,7 @@
 // Forward declarations
 static VALUE configuration_new(VALUE klass, VALUE json_str);
 static void configuration_free(void *ptr);
+static VALUE configuration_get_observe_full_evaluation_data(VALUE self);
 static VALUE configuration_get_assignment(
   VALUE self, VALUE flag_key, VALUE expected_type, VALUE context);
 
@@ -94,6 +95,7 @@ void feature_flags_init(VALUE core_module) {
   rb_undef_alloc_func(configuration_class);
   rb_define_singleton_method(configuration_class, "new", configuration_new, 1);
   rb_define_method(configuration_class, "get_assignment", configuration_get_assignment, 3);
+  rb_define_method(configuration_class, "observe_full_evaluation_data", configuration_get_observe_full_evaluation_data, 0);
 
   rb_gc_register_address(&resolution_details_class);
   resolution_details_class = rb_define_class_under(feature_flags_module, "ResolutionDetails", rb_cObject);
@@ -137,6 +139,12 @@ static VALUE configuration_new(VALUE klass, VALUE json_str) {
   return TypedData_Wrap_Struct(klass, &configuration_data_type, result.ok);
 }
 
+static VALUE configuration_get_observe_full_evaluation_data(VALUE self) {
+  ddog_ffe_Handle_Configuration configuration =
+    (ddog_ffe_Handle_Configuration)rb_check_typeddata(self, &configuration_data_type);
+  return ddog_ffe_configuration_get_observe_full_evaluation_data(configuration) ? Qtrue : Qfalse;
+}
+
 static void configuration_free(void *ptr) {
   ddog_ffe_Handle_Configuration config = (ddog_ffe_Handle_Configuration)ptr;
   ddog_ffe_configuration_drop(&config);

data/ext/libdatadog_api/init.c

@@ -5,7 +5,9 @@
 #include "crashtracker.h"
 #include "feature_flags.h"
 #include "library_config.h"
+#include "otel_thread_context.h"
 #include "process_discovery.h"
+#include "trace_exporter.h"
 
 void ddsketch_init(VALUE core_module);
 void di_init(VALUE datadog_module);
@@ -23,5 +25,11 @@ void DDTRACE_EXPORT Init_libdatadog_api(void) {
   library_config_init(core_module);
   ddsketch_init(core_module);
   feature_flags_init(core_module);
-  di_init(datadog_module);
+  #ifndef TRUFFLERUBY
+    di_init(datadog_module);
+  #endif
+
+  VALUE tracing_module = rb_define_module_under(datadog_module, "Tracing");
+  trace_exporter_init(tracing_module);
+  otel_thread_context_init(tracing_module);
 }

data/ext/libdatadog_api/otel_thread_context.c

@@ -0,0 +1,274 @@
+#include <ruby.h>
+#include <ruby/thread.h>
+
+#include "datadog_ruby_common.h"
+#include "otel_thread_context.h"
+
+// Relies on a :fiber_switch TracePoint that TruffleRuby does not support yet
+#if defined(__linux__) && !defined(TRUFFLERUBY)
+  #define OTEL_THREAD_CONTEXT_SUPPORTED
+#endif
+
+#ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+  #include <datadog/otel-thread-ctx.h>
+  extern __thread const uint8_t *otel_thread_ctx_v1;
+
+  static void otel_ctx_handle_free(void *handle) {
+    ddog_otel_thread_ctx_free((ddog_ThreadContextHandle *) handle);
+  }
+
+  static const rb_data_type_t otel_ctx_handle_t = {
+    .wrap_struct_name = "Datadog::Tracing::OTelThreadContext handle",
+    .function = {.dfree = otel_ctx_handle_free},
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+  };
+#endif
+
+static ID fiber_context_slot;
+
+DDTRACE_UNUSED static bool otel_context_enabled = false;
+
+static VALUE native_set(VALUE _self, VALUE trace_id, VALUE span_id, VALUE local_root_span_id);
+static VALUE native_clear(VALUE _self);
+static VALUE native_enable(VALUE _self);
+static VALUE native_read(VALUE _self);
+
+void otel_thread_context_init(VALUE tracing_module) {
+  fiber_context_slot = rb_intern("__dd_otel_fiber_context");
+
+  VALUE otel_thread_context_class = rb_define_class_under(tracing_module, "OTelThreadContext", rb_cObject);
+
+  rb_define_method(otel_thread_context_class, "_native_enable", native_enable, 0);
+  rb_define_method(otel_thread_context_class, "_native_set", native_set, 3);
+  rb_define_method(otel_thread_context_class, "_native_clear", native_clear, 0);
+
+  VALUE testing_module = rb_define_module_under(otel_thread_context_class, "Testing");
+  rb_define_singleton_method(testing_module, "_native_read", native_read, 0);
+}
+
+#ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+  static ddog_ThreadContextHandle *get_fiber_handle_for(VALUE thread) {
+    VALUE obj = rb_thread_local_aref(thread, fiber_context_slot);
+
+    if (!rb_typeddata_is_kind_of(obj, &otel_ctx_handle_t)) return NULL;
+
+    return RTYPEDDATA_DATA(obj);
+  }
+
+  static ddog_ThreadContextHandle *get_current_fiber_handle(void) {
+    return get_fiber_handle_for(rb_thread_current());
+  }
+
+  static void pack_span_id(VALUE id, uint8_t bytes[8]) {
+    // Tracing::Utils.next_id caps generated span IDs at (1 << 62) - 1, which fits
+    // in a Fixnum on 64-bit Ruby. Packing them directly avoids rb_integer_pack's
+    // generic conversion and format checks on each context update.
+    // https://github.com/DataDog/dd-trace-rb/blob/0b10368e0a2940c1f4d50c9739b9973b4f8bc32a/lib/datadog/tracing/utils.rb#L20-L37
+    if (FIXNUM_P(id)) {
+      uint64_t number = (uint64_t) FIX2LONG(id);
+      for (int i = 7; i >= 0; i--) {
+        bytes[i] = (uint8_t) number;
+        number >>= 8;
+      }
+    } else {
+      rb_integer_pack(id, bytes, 1, sizeof(uint64_t), 0, INTEGER_PACK_BIG_ENDIAN);
+    }
+  }
+
+  static void store_current_fiber_handle(ddog_ThreadContextHandle *handle) {
+    VALUE obj = TypedData_Wrap_Struct(rb_cObject, &otel_ctx_handle_t, handle);
+    rb_thread_local_aset(rb_thread_current(), fiber_context_slot, obj);
+  }
+
+  #ifdef RUBY_INTERNAL_THREAD_EVENT_EXITED
+    static void on_thread_exited(
+      DDTRACE_UNUSED rb_event_flag_t event,
+      DDTRACE_UNUSED const rb_internal_thread_event_data_t *event_data,
+      DDTRACE_UNUSED void *user_data
+    ) {
+      ddog_otel_thread_ctx_detach();
+    }
+  #else
+    static void on_thread_end(
+      DDTRACE_UNUSED rb_event_flag_t evflag,
+      DDTRACE_UNUSED VALUE data,
+      DDTRACE_UNUSED VALUE self,
+      DDTRACE_UNUSED ID mid,
+      DDTRACE_UNUSED VALUE klass
+    ) {
+      ddog_otel_thread_ctx_detach();
+    }
+
+    // CRuby fires RUBY_EVENT_THREAD_END only after the thread body returns,
+    // and Thread#kill unwinds the body via EC_JUMP_TAG(TAG_FATAL):
+    // https://github.com/ruby/ruby/blob/v3_1_4/thread.c#L744-L775
+    static void on_thread_begin(
+      DDTRACE_UNUSED rb_event_flag_t evflag,
+      DDTRACE_UNUSED VALUE data,
+      DDTRACE_UNUSED VALUE self,
+      DDTRACE_UNUSED ID mid,
+      DDTRACE_UNUSED VALUE klass
+    ) {
+      ddog_otel_thread_ctx_detach();
+    }
+  #endif
+
+  #ifdef HAVE_RUBY_THREAD_STORAGE_API
+    // This function is declared in internal/thread.h.
+    // It returns 0 when the current thread sits in a blocking region.
+    int ruby_thread_has_gvl_p(void);
+
+    static bool in_blocking_region(void) {
+      #ifdef HAVE_RUBY_THREAD_HAS_GVL_P
+        return ruby_thread_has_gvl_p() == 0;
+      #else
+        return false;
+      #endif
+    }
+
+    // Under M:N a Ruby thread can migrate between native threads,
+    // so we need to detach the context when the Ruby thread gets suspended.
+    static void on_thread_suspended(
+      DDTRACE_UNUSED rb_event_flag_t event,
+      DDTRACE_UNUSED const rb_internal_thread_event_data_t *event_data,
+      DDTRACE_UNUSED void *user_data
+    ) {
+      // Ruby pins threads that release GVL to call into a native extension,
+      // so we know the thread will not migrate.
+      if (in_blocking_region()) return;
+
+      ddog_otel_thread_ctx_detach();
+    }
+
+    static void on_thread_resumed(
+      DDTRACE_UNUSED rb_event_flag_t event,
+      const rb_internal_thread_event_data_t *event_data,
+      DDTRACE_UNUSED void *user_data
+    ) {
+      ddog_ThreadContextHandle *handle = get_fiber_handle_for(event_data->thread);
+
+      if (handle != NULL) ddog_otel_thread_ctx_attach(handle);
+    }
+  #endif
+
+  static void on_fiber_switch(
+    DDTRACE_UNUSED rb_event_flag_t evflag,
+    DDTRACE_UNUSED VALUE data,
+    DDTRACE_UNUSED VALUE self,
+    DDTRACE_UNUSED ID mid,
+    DDTRACE_UNUSED VALUE klass
+  ) {
+    ddog_ThreadContextHandle *handle = get_current_fiber_handle();
+
+    if (handle != NULL) {
+      ddog_otel_thread_ctx_attach(handle);
+    } else {
+      ddog_otel_thread_ctx_detach();
+    }
+  }
+#endif
+
+static VALUE native_enable(DDTRACE_UNUSED VALUE _self) {
+  #ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+    if (otel_context_enabled) return Qtrue;
+    otel_context_enabled = true;
+
+    #ifdef RUBY_INTERNAL_THREAD_EVENT_EXITED
+      rb_internal_thread_add_event_hook(on_thread_exited, RUBY_INTERNAL_THREAD_EVENT_EXITED, NULL);
+    #else
+      rb_add_event_hook(on_thread_end, RUBY_EVENT_THREAD_END, Qnil);
+      // RUBY_EVENT_THREAD_END does not fire for threads terminated via Thread#kill
+      // on Ruby < 3.2, so a killed thread's otel_thread_ctx_v1 TLS stays attached
+      // on a native thread CRuby may recycle for a later Ruby thread. Detach on
+      // thread begin to drop that stale record before the new thread runs.
+      rb_add_event_hook(on_thread_begin, RUBY_EVENT_THREAD_BEGIN, Qnil);
+    #endif
+
+    // These hooks only matter under the M:N thread scheduler (Ruby 3.3+),
+    // where a Ruby thread can migrate between native threads.
+    #ifdef HAVE_RUBY_THREAD_STORAGE_API
+      rb_internal_thread_add_event_hook(on_thread_suspended, RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, NULL);
+      rb_internal_thread_add_event_hook(on_thread_resumed, RUBY_INTERNAL_THREAD_EVENT_RESUMED, NULL);
+    #endif
+
+    rb_add_event_hook(on_fiber_switch, RUBY_EVENT_FIBER_SWITCH, Qnil);
+
+    return Qtrue;
+  #else
+    raise_error(rb_eRuntimeError, "OTel thread context is not supported on this platform");
+  #endif
+}
+
+static VALUE native_set(
+    DDTRACE_UNUSED VALUE _self,
+    DDTRACE_UNUSED VALUE trace_id,
+    DDTRACE_UNUSED VALUE span_id,
+    DDTRACE_UNUSED VALUE local_root_span_id
+  ) {
+  #ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+    if (!otel_context_enabled) return Qfalse;
+
+    uint8_t trace_id_bytes[16];
+    uint8_t span_id_bytes[8];
+    uint8_t local_root_span_id_bytes[8];
+
+    // Note: We use 2 words of size 8, as `rb_integer_pack` has a fast path for it (vs 16 bytes of size 1)
+    rb_integer_pack(trace_id, trace_id_bytes, 2, sizeof(uint64_t), 0, INTEGER_PACK_BIG_ENDIAN);
+    pack_span_id(span_id, span_id_bytes);
+    pack_span_id(local_root_span_id, local_root_span_id_bytes);
+
+    ddog_ThreadContextHandle *handle = get_current_fiber_handle();
+
+    if (handle == NULL) {
+      handle = ddog_otel_thread_ctx_new(&trace_id_bytes, &span_id_bytes, 0, &local_root_span_id_bytes);
+      store_current_fiber_handle(handle);
+      ddog_otel_thread_ctx_attach(handle);
+    } else {
+      ddog_otel_thread_ctx_update_and_attach(handle, &trace_id_bytes, &span_id_bytes, 0, &local_root_span_id_bytes);
+    }
+
+    return Qtrue;
+  #else
+    return Qfalse;
+  #endif
+}
+
+static VALUE native_clear(DDTRACE_UNUSED VALUE _self) {
+  #ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+    if (!otel_context_enabled) return Qfalse;
+
+    ddog_ThreadContextHandle *detached = ddog_otel_thread_ctx_detach();
+    rb_thread_local_aset(rb_thread_current(), fiber_context_slot, Qnil);
+
+    return detached == NULL ? Qfalse : Qtrue;
+  #else
+    return Qfalse;
+  #endif
+}
+
+static VALUE native_read(DDTRACE_UNUSED VALUE _self) {
+  #ifdef OTEL_THREAD_CONTEXT_SUPPORTED
+    const uint8_t *raw = otel_thread_ctx_v1;
+    if (!raw) return Qnil;
+
+    uint16_t attrs_data_size = (uint16_t) raw[26] | ((uint16_t) raw[27] << 8);
+
+    if (attrs_data_size > ddog_MAX_ATTRS_DATA_SIZE) {
+      raise_error(
+        rb_eRuntimeError,
+        "Invalid OTel thread context record: attrs_data_size (%d) exceeds maximum (%d)",
+        attrs_data_size, ddog_MAX_ATTRS_DATA_SIZE
+      );
+    }
+
+    VALUE result = rb_hash_new();
+    rb_hash_aset(result, ID2SYM(rb_intern("trace_id")), rb_str_new((const char *) raw, 16));
+    rb_hash_aset(result, ID2SYM(rb_intern("span_id")), rb_str_new((const char *) (raw + 16), 8));
+    rb_hash_aset(result, ID2SYM(rb_intern("valid")), rb_str_new((const char *) (raw + 24), 1));
+    rb_hash_aset(result, ID2SYM(rb_intern("attrs")), rb_str_new((const char *) (raw + 28), attrs_data_size));
+
+    return result;
+  #else
+    return Qnil;
+  #endif
+}

data/ext/libdatadog_api/otel_thread_context.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <ruby.h>
+
+void otel_thread_context_init(VALUE tracing_module);

data/ext/libdatadog_api/process_discovery.c

@@ -4,6 +4,7 @@
 #include <datadog/library-config.h>
 
 #include "datadog_ruby_common.h"
+#include "helpers.h"
 
 static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self);
 static VALUE _native_to_rb_int(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd);
@@ -38,8 +39,9 @@ void process_discovery_init(VALUE core_module) {
 
 static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, VALUE self) {
   VALUE logger;
+  VALUE otel_thread_context_enabled;
   VALUE options;
-  rb_scan_args(argc, argv, "1:", &logger, &options);
+  rb_scan_args(argc, argv, "2:", &logger, &otel_thread_context_enabled, &options);
   if (options == Qnil) options = rb_hash_new();
 
   VALUE runtime_id = rb_hash_fetch(options, ID2SYM(rb_intern("runtime_id")));
@@ -61,6 +63,7 @@ static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, VALUE self) {
   ENFORCE_TYPE(service_version, T_STRING);
   ENFORCE_TYPE(process_tags, T_STRING);
   ENFORCE_TYPE(container_id, T_STRING);
+  ENFORCE_BOOLEAN(otel_thread_context_enabled);
 
   void* builder = ddog_tracer_metadata_new();
 
@@ -74,6 +77,16 @@ static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, VALUE self) {
   ddog_tracer_metadata_set(builder, DDOG_METADATA_KIND_PROCESS_TAGS, StringValueCStr(process_tags));
   ddog_tracer_metadata_set(builder, DDOG_METADATA_KIND_CONTAINER_ID, StringValueCStr(container_id));
 
+  if (RTEST(otel_thread_context_enabled)) {
+    ddog_VoidResult include_result = ddog_tracer_metadata_include_otel_thread_context(builder);
+
+    if (include_result.tag == DDOG_VOID_RESULT_ERR) {
+      ddog_tracer_metadata_free(builder);
+    }
+
+    CHECK_VOID_RESULT("Failed to include OTel thread context metadata", include_result);
+  }
+
   ddog_Result_TracerMemfdHandle result = ddog_tracer_metadata_store(builder);
   ddog_tracer_metadata_free(builder);
 
@@ -89,6 +102,7 @@ static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, VALUE self) {
   *fd = result.ok.fd;
   VALUE tracer_memfd_class = rb_const_get(self, rb_intern("TracerMemfd"));
   VALUE tracer_memfd = TypedData_Wrap_Struct(tracer_memfd_class, &tracer_memfd_type, fd);
+
   return tracer_memfd;
 }