datadog 2.30.0 → 2.43.0

data/ext/datadog_profiling_native_extension/heap_recorder.h

@@ -80,9 +80,9 @@ void heap_recorder_set_size_enabled(heap_recorder *heap_recorder, bool size_enab
 // Controls how many recordings will be ignored before committing a heap allocation and
 // the weight of the committed heap allocation.
 //
-// A value of 1 will effectively track all objects that are passed through
-// start/end_heap_allocation_recording pairs. A value of 10 will only track every 10th
-// object passed through such calls and its effective weight for the purposes of heap
+// A value of 1 will effectively track all objects that are passed to
+// `heap_recorder_record_allocation`. A value of 10 will only track every 10th
+// object passed to it and its effective weight for the purposes of heap
 // profiling will be multiplied by 10.
 //
 // NOTE: Default is 1, i.e., track all heap allocation recordings.
@@ -94,45 +94,47 @@ void heap_recorder_set_sample_rate(heap_recorder *heap_recorder, int sample_rate
 // WARN: Assumes this gets called before profiler is reinitialized on the fork
 void heap_recorder_after_fork(heap_recorder *heap_recorder);
 
-// Start a heap allocation recording on the heap recorder for a new object.
+// Record a heap allocation on the heap recorder.
 //
-// This heap allocation recording needs to be ended via ::end_heap_allocation_recording
-// before it will become fully committed and able to be iterated on.
+// The recording is added to a pending list, and only becomes fully tracked and able to be iterated on once
+// `heap_recorder_commit_recordings_may_lose_gvl` gets to it.
 //
-// @param new_obj
+// @param new_object
 //   The newly allocated Ruby object/value.
 // @param weight
 //   The sampling weight of this object.
-//
-// WARN: It needs to be paired with a ::end_heap_allocation_recording call.
-// Returns needs_after_allocation: true whenever the pending_recordings buffer goes from empty to non-empty and thus
-// a after_sample callback is required to flush it
-bool start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj, unsigned int weight, ddog_CharSlice alloc_class);
-
-// End a previously started heap allocation recording on the heap recorder.
-//
-// It is at this point that an allocated object will become fully tracked and able to be iterated on.
-//
-// @param locations The stacktrace representing the location of the allocation.
-//
-// WARN: It is illegal to call this without previously having called ::start_heap_allocation_recording.
-// WARN: This method rescues exceptions with `rb_protect`, returning the exception state integer for the caller to handle.
-__attribute__((warn_unused_result))
-int end_heap_allocation_recording_with_rb_protect(heap_recorder *heap_recorder, ddog_prof_Slice_Location locations);
+// @param alloc_class
+//   The class of the newly allocated object.
+// @param locations
+//   The stacktrace representing the location of the allocation.
+// @param needs_commit
+//   Out: set to whether there are pending recordings, and thus a `heap_recorder_commit_recordings_may_lose_gvl`
+//   callback is needed to commit them. Note that we answer this on every call, including the ones where we don't
+//   record anything: asking again is how we recover if a commit gets skipped for any reason.
+//
+// WARN: This gets called from inside the RUBY_INTERNAL_EVENT_NEWOBJ tracepoint, so it neither allocates in the Ruby
+// heap nor releases the GVL (https://github.com/DataDog/dd-trace-rb/pull/4240).
+void heap_recorder_record_allocation(
+  heap_recorder *heap_recorder,
+  VALUE new_object,
+  unsigned int weight,
+  ddog_CharSlice alloc_class,
+  ddog_prof_Slice_Location locations,
+  bool *needs_commit
+);
 
 // Update the heap recorder, **checking young objects only**. The idea here is to align with GC: most young objects never
 // survive enough GC generations, and thus periodically running this method reduces memory usage (we get rid of
 // these objects quicker) and hopefully reduces tail latency (because there's less objects at serialization time to check).
 void heap_recorder_update_young_objects(heap_recorder *heap_recorder);
 
-// Finalize any pending heap allocation recordings by getting their object IDs.
+// Commit any pending heap allocation recordings by taking a weak reference to their objects.
 // This should be called via a postponed job, after the on_newobj_event has completed.
-// Raises an exception if a fatal error occurs (e.g., bignum object ID detected).
-void heap_recorder_finalize_pending_recordings(heap_recorder *heap_recorder);
+void heap_recorder_commit_recordings_may_lose_gvl(heap_recorder *heap_recorder);
 
-// Mark pending recordings to prevent GC from collecting the objects
-// while they're waiting for the recordings to be finalized.
-void heap_recorder_mark_pending_recordings(heap_recorder *heap_recorder);
+// Mark the Ruby objects the heap recorder holds on to: the objects of any pending recordings (so that GC does not
+// collect them while they're waiting to be committed) and the weak map itself.
+void heap_recorder_mark(heap_recorder *heap_recorder);
 
 // Update the heap recorder to reflect the latest state of the VM and prepare internal structures
 // for efficient iteration.
@@ -179,10 +181,27 @@ VALUE heap_recorder_state_snapshot(heap_recorder *heap_recorder);
 // troubleshoot issues such as unexpected test failures.
 VALUE heap_recorder_testonly_debug(heap_recorder *heap_recorder);
 
-// Check if a given object_id is being tracked or not
-VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, VALUE obj_id);
+// Check if a given record_id is being tracked or not
+VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, long record_id);
+
+// Returns the record id being used to track the given (still alive) object, or nil if it's not being tracked (which
+// includes the case where heap profiling is disabled). This lets tests hold on to a record id and keep asking about
+// it after the object itself is gone.
+// NOTE: This is a linear scan over the tracked objects; fine for the small numbers tests track.
+VALUE heap_recorder_testonly_record_id_for(heap_recorder *heap_recorder, VALUE obj);
+
+// Overrides the GC gen a tracked object is considered to have been allocated in. This lets tests give an object an
+// exact age, instead of relying on an exact number of GCs having run.
+void heap_recorder_testonly_set_object_alloc_gen(heap_recorder *heap_recorder, long record_id, size_t alloc_gen);
 
 // Used to ensure that a GC actually triggers an update of the objects
 void heap_recorder_testonly_reset_last_update(heap_recorder *heap_recorder);
 
+// Exhausts the usable record ids, so that all subsequent heap recordings raise (the id keeps being incremented, so
+// there's no going back). Used to test that our callers correctly handle a heap recorder that raises in the middle of
+// recording a sample.
+//
+// WARN: This should only be used for testing.
+void heap_recorder_testonly_exhaust_record_ids(heap_recorder *heap_recorder);
+
 void heap_recorder_testonly_benchmark_intern(heap_recorder *heap_recorder, ddog_CharSlice string, int times, bool use_all);

data/ext/datadog_profiling_native_extension/http_transport.c

@@ -198,15 +198,15 @@ static VALUE _native_do_export(
   VALUE flush
 ) {
   VALUE encoded_profile = rb_funcall(flush, rb_intern("encoded_profile"), 0);
-  VALUE code_provenance_file_name = rb_funcall(flush, rb_intern("code_provenance_file_name"), 0);
   VALUE code_provenance_data = rb_funcall(flush, rb_intern("code_provenance_data"), 0);
+  VALUE metrics_json = rb_funcall(flush, rb_intern("metrics"), 0);
   VALUE tags_as_array = rb_funcall(flush, rb_intern("tags_as_array"), 0);
   VALUE internal_metadata_json = rb_funcall(flush, rb_intern("internal_metadata_json"), 0);
   VALUE info_json = rb_funcall(flush, rb_intern("info_json"), 0);
   VALUE process_tags = rb_funcall(flush, rb_intern("process_tags"), 0);
 
   enforce_encoded_profile_instance(encoded_profile);
-  ENFORCE_TYPE(code_provenance_file_name, T_STRING);
+  ENFORCE_TYPE(metrics_json, T_STRING);
   ENFORCE_TYPE(tags_as_array, T_ARRAY);
   ENFORCE_TYPE(internal_metadata_json, T_STRING);
   ENFORCE_TYPE(info_json, T_STRING);
@@ -216,13 +216,18 @@ static VALUE _native_do_export(
   bool have_code_provenance = !NIL_P(code_provenance_data);
   if (have_code_provenance) ENFORCE_TYPE(code_provenance_data, T_STRING);
 
-  int to_compress_length = have_code_provenance ? 1 : 0;
+  int to_compress_length = 1 + (have_code_provenance ? 1 : 0);
   ddog_prof_Exporter_File to_compress[to_compress_length];
   ddog_prof_Exporter_Slice_File files_to_compress_and_export = {.ptr = to_compress, .len = to_compress_length};
 
+  to_compress[0] = (ddog_prof_Exporter_File) {
+    .name = DDOG_CHARSLICE_C("metrics.json"),
+    .file = byte_slice_from_ruby_string(metrics_json),
+  };
+
   if (have_code_provenance) {
-    to_compress[0] = (ddog_prof_Exporter_File) {
-      .name = char_slice_from_ruby_string(code_provenance_file_name),
+    to_compress[1] = (ddog_prof_Exporter_File) {
+      .name = DDOG_CHARSLICE_C("code-provenance.json"),
       .file = byte_slice_from_ruby_string(code_provenance_data),
     };
   }

data/ext/datadog_profiling_native_extension/macos_sampler_thread.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <mach/thread_policy.h>
+#include <pthread.h>
+#include <stdio.h>
+
+#include "time_helpers.h"
+
+// On macOS the default scheduler wake latency for a timeshare thread is ~1-2ms,
+// so nanosleep regularly overshoots a 10ms request by ~2ms. Marking the worker
+// thread with THREAD_TIME_CONSTRAINT_POLICY tells the scheduler this thread has
+// a periodic deadline; the kernel then wakes it close to the requested time.
+//
+// This only affects the sampler worker thread. We pair it with a demote call
+// before the thread can be reused by Ruby for unrelated Ruby threads (see the
+// SIGPROF unblock dance in _native_sampling_loop).
+static inline void promote_sampler_thread_to_realtime(uint64_t period_ns) {
+  mach_timebase_info_data_t timebase = (mach_timebase_info_data_t) {};
+  mach_timebase_info(&timebase);
+
+  // ns -> mach ticks: each mach tick is (numer/denom) ns, so divide by that.
+  #define NS_TO_MACH_TICKS(ns) ((uint32_t) (((uint64_t)(ns) * (uint64_t) timebase.denom) / (uint64_t) timebase.numer))
+
+  struct thread_time_constraint_policy policy = {
+    .period      = NS_TO_MACH_TICKS(period_ns),
+    .computation = NS_TO_MACH_TICKS(MICROS_AS_NS(200)),  // 200us upper bound on the work we do per period
+    .constraint  = NS_TO_MACH_TICKS(MILLIS_AS_NS(1)),    // wake us within 1ms of the deadline
+    .preemptible = TRUE,
+  };
+
+  #undef NS_TO_MACH_TICKS
+
+  kern_return_t kr = thread_policy_set(
+    pthread_mach_thread_np(pthread_self()),
+    THREAD_TIME_CONSTRAINT_POLICY,
+    (thread_policy_t) &policy,
+    THREAD_TIME_CONSTRAINT_POLICY_COUNT
+  );
+  if (kr != KERN_SUCCESS) {
+    // Non-fatal: we'll fall back to the default scheduler behavior (overshoot ~2ms).
+    fprintf(stderr, "[ddtrace] Failed to set real-time policy on profiler sampler thread (kr=%d); sample cadence may be imprecise\n", kr);
+  }
+}
+
+static inline void demote_sampler_thread_from_realtime(void) {
+  struct thread_standard_policy policy = {.no_data = 0};
+  thread_policy_set(
+    pthread_mach_thread_np(pthread_self()),
+    THREAD_STANDARD_POLICY,
+    (thread_policy_t) &policy,
+    THREAD_STANDARD_POLICY_COUNT
+  );
+}

data/ext/datadog_profiling_native_extension/native_extension_helpers.rb

@@ -9,9 +9,6 @@ module Datadog
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = "DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION"
 
-      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on datadog-ruby_core_source
-      CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
-
       def self.fail_install_if_missing_extension?
         ENV[ENV_FAIL_INSTALL_IF_MISSING_EXTENSION].to_s.strip.downcase == "true"
       end
@@ -35,7 +32,6 @@ module Datadog
             on_macos? ||
             on_unknown_os? ||
             on_unsupported_cpu_arch? ||
-            expected_to_use_mjit_but_mjit_is_disabled? ||
             libdatadog_not_available? ||
             libdatadog_not_usable?
         end
@@ -80,15 +76,15 @@ module Datadog
           "For help solving this issue, please contact Datadog support at",
           "<https://docs.datadoghq.com/help/>.",
           "You can also check out the Continuous Profiler troubleshooting page at",
-          "<https://dtdg.co/ruby-profiler-troubleshooting>."
+          "<https://dtdg.co/ruby-profiler-troubleshooting>.",
         ].freeze
 
         GET_IN_TOUCH = [
-          "Get in touch with us if you're interested in profiling your app!"
+          "Get in touch with us if you're interested in profiling your app!",
         ].freeze
 
         UPGRADE_RUBY = [
-          "Upgrade to a modern Ruby to enable profiling for your app."
+          "Upgrade to a modern Ruby to enable profiling for your app.",
         ].freeze
 
         # Validation for this check is done in extconf.rb because it relies on mkmf
@@ -97,13 +93,6 @@ module Datadog
           suggested: CONTACT_SUPPORT,
         )
 
-        # Validation for this check is done in extconf.rb because it relies on mkmf
-        COMPILATION_BROKEN = explain_issue(
-          "compilation of the Ruby VM just-in-time header failed.",
-          "Your C compiler or Ruby VM just-in-time compiler seem to be broken.",
-          suggested: CONTACT_SUPPORT,
-        )
-
         # Validation for this check is done in extconf.rb because it relies on mkmf
         COMPILER_ATOMIC_MISSING = explain_issue(
           "your C compiler is missing support for the <stdatomic.h> header.",
@@ -187,19 +176,6 @@ module Datadog
           architecture_not_supported unless RUBY_PLATFORM.start_with?("x86_64", "aarch64", "arm64")
         end
 
-        # On some Rubies, we require the mjit header to be present. If Ruby was installed without MJIT support, we also skip
-        # building the extension.
-        private_class_method def self.expected_to_use_mjit_but_mjit_is_disabled?
-          ruby_without_mjit = explain_issue(
-            "your Ruby has been compiled without JIT support (--disable-jit-support).",
-            "The profiling native extension requires a Ruby compiled with JIT support,",
-            "even if the JIT is not in use by the application itself.",
-            suggested: CONTACT_SUPPORT,
-          )
-
-          ruby_without_mjit if CAN_USE_MJIT_HEADER && RbConfig::CONFIG["MJIT_SUPPORT"] != "yes"
-        end
-
         private_class_method def self.libdatadog_not_available?
           Datadog::LibdatadogExtconfHelpers.try_loading_libdatadog do |exception|
             explain_issue(