ddtrace 1.20.0 → 1.22.0

data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.c

@@ -9,16 +9,23 @@
 #define BASE_SAMPLING_INTERVAL 50
 
 #define ADJUSTMENT_WINDOW_NS SECONDS_AS_NS(1)
+#define ADJUSTMENT_WINDOW_SAMPLES 100
+// Any average sampling times above this value will be clamped to this value.
+// In practice, this limits the budget consumption of a single sample to that of an adjustment window,
+// thus aiming for a minimum sample rate of once per adjustment window (dependent on actual event rate).
+// NOTE: This is our main strategy to deal with timing hiccups such as those that can be caused by
+//       suspensions, system overloads and other things that could lead to arbitrarily big sampling
+//       time measurements.
+#define MAX_ALLOWED_SAMPLING_NS(target_overhead) (long) (ADJUSTMENT_WINDOW_NS * target_overhead / 100.)
 
 #define EMA_SMOOTHING_FACTOR 0.6
-#define EXP_MOVING_AVERAGE(last, avg, first) first ? last : (1-EMA_SMOOTHING_FACTOR) * avg + EMA_SMOOTHING_FACTOR * last
 
-void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name) {
+void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name, long now_ns) {
   sampler->debug_name = debug_name;
-  discrete_dynamic_sampler_set_overhead_target_percentage(sampler, BASE_OVERHEAD_PCT);
+  discrete_dynamic_sampler_set_overhead_target_percentage(sampler, BASE_OVERHEAD_PCT, now_ns);
 }
 
-static void _discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, long now_ns) {
+void discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, long now_ns) {
   const char *debug_name = sampler->debug_name;
   double target_overhead = sampler->target_overhead;
   (*sampler) = (discrete_dynamic_sampler) {
@@ -31,6 +38,7 @@ static void _discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, l
     // This fake readjustment will use a hardcoded sampling interval
     .sampling_interval = BASE_SAMPLING_INTERVAL,
     .sampling_probability = 1.0 / BASE_SAMPLING_INTERVAL,
+    .max_sampling_time_ns = MAX_ALLOWED_SAMPLING_NS(target_overhead),
     // But we want to make sure we sample at least once in the next window so that our first
     // real readjustment has some notion of how heavy sampling is. Therefore, we'll make it so that
     // the next event is automatically sampled by artificially locating it in the interval threshold.
@@ -38,27 +46,17 @@ static void _discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, l
   };
 }
 
-void discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler) {
-  long now = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
-  _discrete_dynamic_sampler_reset(sampler, now);
-}
-
-static void _discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead, long now_ns) {
+void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead, long now_ns) {
   if (target_overhead <= 0 || target_overhead > 100) {
     rb_raise(rb_eArgError, "Target overhead must be a double between ]0,100] was %f", target_overhead);
   }
   sampler->target_overhead = target_overhead;
-  _discrete_dynamic_sampler_reset(sampler, now_ns);
-}
-
-void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead) {
-  long now = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
-  _discrete_dynamic_sampler_set_overhead_target_percentage(sampler, target_overhead, now);
+  return discrete_dynamic_sampler_reset(sampler, now_ns);
 }
 
 static void maybe_readjust(discrete_dynamic_sampler *sampler, long now);
 
-static bool _discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler, long now_ns) {
+bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler, long now_ns) {
   // For efficiency reasons we don't do true random sampling but rather systematic
   // sampling following a sample interval/skip. This can be biased and hide patterns
   // but the dynamic interval and rather indeterministic pattern of allocations in
@@ -77,12 +75,7 @@ static bool _discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sa
   return should_sample;
 }
 
-bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler) {
-  long now = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
-  return _discrete_dynamic_sampler_should_sample(sampler, now);
-}
-
-static long _discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler, long now_ns) {
+long discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler, long now_ns) {
   long last_sampling_time_ns = sampler->sample_start_time_ns == 0 ? 0 : long_max_of(0, now_ns - sampler->sample_start_time_ns);
   sampler->samples_since_last_readjustment++;
   sampler->sampling_time_since_last_readjustment_ns += last_sampling_time_ns;
@@ -94,11 +87,6 @@ static long _discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sam
   return last_sampling_time_ns;
 }
 
-long discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler) {
-  long now = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
-  return _discrete_dynamic_sampler_after_sample(sampler, now);
-}
-
 double discrete_dynamic_sampler_probability(discrete_dynamic_sampler *sampler) {
   return sampler->sampling_probability * 100.;
 }
@@ -107,35 +95,66 @@ size_t discrete_dynamic_sampler_events_since_last_sample(discrete_dynamic_sample
   return sampler->events_since_last_sample;
 }
 
+static double ewma_adj_window(double latest_value, double avg, long current_window_time_ns, bool is_first) {
+  if (is_first) {
+    return latest_value;
+  }
+
+  // We don't want samples coming from partial adjustment windows (e.g. preempted due to number of samples)
+  // to lead to quick "forgetting" of the past. Thus, we'll tweak the weight of this new value based on the
+  // size of the time window from which we gathered it in relation to our standard adjustment window time.
+  double fraction_of_full_window = double_min_of((double) current_window_time_ns / ADJUSTMENT_WINDOW_NS, 1);
+  double alpha = EMA_SMOOTHING_FACTOR * fraction_of_full_window;
+
+  return (1-alpha) * avg + alpha * latest_value;
+}
+
 static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
-  long window_time_ns = sampler->last_readjust_time_ns == 0 ? ADJUSTMENT_WINDOW_NS : now - sampler->last_readjust_time_ns;
+  long this_window_time_ns = sampler->last_readjust_time_ns == 0 ? ADJUSTMENT_WINDOW_NS : now - sampler->last_readjust_time_ns;
+
+  bool should_readjust_based_on_time = this_window_time_ns >= ADJUSTMENT_WINDOW_NS;
+  bool should_readjust_based_on_samples = sampler->samples_since_last_readjustment >= ADJUSTMENT_WINDOW_SAMPLES;
+
+  if (!should_readjust_based_on_time && !should_readjust_based_on_samples) {
+    // not enough time or samples have passed to perform a readjustment
+    return;
+  }
 
-  if (window_time_ns < ADJUSTMENT_WINDOW_NS) {
-    // not enough time has passed to perform a readjustment
+  if (this_window_time_ns == 0) {
+    // should not be possible given previous condition but lets protect against div by 0 below.
     return;
   }
 
   // If we got this far, lets recalculate our sampling params based on new observations
   bool first_readjustment = !sampler->has_completed_full_adjustment_window;
 
-  // Update our running average of events/sec with latest observation
-  sampler->events_per_ns = EXP_MOVING_AVERAGE(
-    (double) sampler->events_since_last_readjustment / window_time_ns,
+  // Update our running average of events/sec with latest observation.
+  sampler->events_per_ns = ewma_adj_window(
+    (double) sampler->events_since_last_readjustment / this_window_time_ns,
     sampler->events_per_ns,
+    this_window_time_ns,
     first_readjustment
   );
 
   // Update our running average of sampling time for a specific event
-  long sampling_window_time_ns = sampler->sampling_time_since_last_readjustment_ns;
-  long sampling_overshoot_time_ns = -1;
   if (sampler->samples_since_last_readjustment > 0) {
     // We can only update sampling-related stats if we actually sampled on the last window...
 
     // Lets update our average sampling time per event
-    long avg_sampling_time_in_window_ns = sampler->samples_since_last_readjustment == 0 ? 0 : sampling_window_time_ns / sampler->samples_since_last_readjustment;
-    sampler->sampling_time_ns = EXP_MOVING_AVERAGE(
+    long avg_sampling_time_in_window_ns = sampler->samples_since_last_readjustment == 0 ? 0 : sampler->sampling_time_since_last_readjustment_ns / sampler->samples_since_last_readjustment;
+    if (avg_sampling_time_in_window_ns > sampler->max_sampling_time_ns) {
+      // If the average sampling time in the previous window was deemed unnacceptable, clamp it to the
+      // maximum acceptable value and register this operation in our counter.
+      // NOTE: This is important so that events like suspensions or system overloads do not lead us to
+      //       learn arbitrarily big sampling times which may then result in us not sampling anything
+      //       for very long periods of time.
+      avg_sampling_time_in_window_ns = sampler->max_sampling_time_ns;
+      sampler->sampling_time_clamps++;
+    }
+    sampler->sampling_time_ns = ewma_adj_window(
       avg_sampling_time_in_window_ns,
       sampler->sampling_time_ns,
+      this_window_time_ns,
       first_readjustment
     );
   }
@@ -145,21 +164,21 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
   // NOTE: Updating this even when no samples occur is a conscious choice which enables us to cooldown extreme adjustments over time.
   //       If we didn't do this, whenever a big spike caused target_overhead_adjustment to equal target_overhead, we'd get stuck
   //       in a "probability = 0" state.
-  long reference_target_sampling_time_ns = window_time_ns * (sampler->target_overhead / 100.);
+  long this_window_sampling_target_time_ns = this_window_time_ns * (sampler->target_overhead / 100.);
   // Overshoot by definition is always >= 0. < 0 would be undershooting!
-  sampling_overshoot_time_ns = long_max_of(0, sampler->sampling_time_since_last_readjustment_ns - reference_target_sampling_time_ns);
+  long this_window_sampling_overshoot_time_ns = long_max_of(0, sampler->sampling_time_since_last_readjustment_ns - this_window_sampling_target_time_ns);
   // Our overhead adjustment should always be between [-target_overhead, 0]. Higher adjustments would lead to negative overhead targets
   // which don't make much sense.
-  double last_target_overhead_adjustment = -double_min_of(sampler->target_overhead, sampling_overshoot_time_ns * 100. / window_time_ns);
-  sampler->target_overhead_adjustment = EXP_MOVING_AVERAGE(
+  double last_target_overhead_adjustment = -double_min_of(sampler->target_overhead, this_window_sampling_overshoot_time_ns * 100. / this_window_time_ns);
+  sampler->target_overhead_adjustment = ewma_adj_window(
     last_target_overhead_adjustment,
     sampler->target_overhead_adjustment,
+    this_window_time_ns,
     first_readjustment
   );
 
   // Apply our overhead adjustment to figure out our real targets for this readjustment.
   double target_overhead = double_max_of(0, sampler->target_overhead + sampler->target_overhead_adjustment);
-  long target_sampling_time_ns = window_time_ns * (target_overhead / 100.);
 
   // Recalculate target sampling probability so that the following 2 hold:
   // * window_time_ns = working_window_time_ns + sampling_window_time_ns
@@ -175,11 +194,13 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
   //                                                ┌─ assuming no events will be emitted during sampling
   //                                                │
   //                           = events_per_ns * working_window_time_ns * sampling_probability * sampling_time_ns
+  //                           = events_per_ns * (window_time_ns - sampling_window_time_ns) * sampling_probability * sampling_time_ns
   //
   // Re-ordering for sampling_probability and solving for the upper-bound of sampling_window_time_ns:
   //
   //   sampling_window_time_ns = window_time_ns * target_overhead / 100
-  //   sampling_probability = window_time_ns * target_overhead / 100 / (events_per_ns * working_window_time_ns * sampling_time_ns) =
+  //   sampling_probability = (sampling_window_time_ns) / (events_per_ns * sampling_time_ns * (window_time_ns - sampling_window_time_ns))
+  //                        = (window_time_ns * target_overhead / 100) / (events_per_ns * sampling_time_ns * window_time_ns * (1 - target_overhead / 100))
   //
   // Which you can intuitively understand as:
   //
@@ -190,16 +211,22 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
   //   then probability will be > 1 (but we should clamp to 1 since probabilities higher than 1 don't make sense).
   // * If app is eventing a lot or our sampling overhead is big, then as time_to_sample_all_events_ns grows, sampling_probability will
   //   tend to 0.
-  long working_window_time_ns = long_max_of(0, window_time_ns - sampling_window_time_ns);
-  double max_allowed_time_for_sampling_ns = target_sampling_time_ns;
-  long time_to_sample_all_events_ns = sampler->events_per_ns * working_window_time_ns * sampler->sampling_time_ns;
-  if (max_allowed_time_for_sampling_ns == 0) {
-    // if we aren't allowed any sampling time at all, probability has to be 0
+  //
+  // In fact, we can simplify the equation further since the `window_time_ns` components cancel each other out:
+  //
+  //   sampling_probability = (target_overhead / 100) / (events_per_ns * sampling_time_ns * (1 - target_overhead / 100))
+  //                        = max_sampling_overhead / avg_sampling_overhead
+
+  double max_sampling_overhead = target_overhead / 100.;
+  double avg_sampling_overhead = sampler->events_per_ns * sampler->sampling_time_ns * (1 - max_sampling_overhead);
+
+  if (max_sampling_overhead == 0) {
+    // if we aren't allowed any sampling overhead at all, probability has to be 0
     sampler->sampling_probability = 0;
   } else {
     // otherwise apply the formula described above (protecting against div by 0)
-    sampler->sampling_probability = time_to_sample_all_events_ns == 0 ? 1. :
-      double_min_of(1., max_allowed_time_for_sampling_ns / time_to_sample_all_events_ns);
+    sampler->sampling_probability = avg_sampling_overhead == 0 ? 1. :
+      double_min_of(1., max_sampling_overhead / avg_sampling_overhead);
   }
 
   // Doing true random selection would involve "tossing a coin" on every allocation. Lets do systematic sampling instead so that our
@@ -225,26 +252,34 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
     double samples_in_60s = allocs_in_60s * sampler->sampling_probability;
     double expected_total_sampling_time_in_60s =
       samples_in_60s * sampler->sampling_time_ns / 1e9;
-    double real_total_sampling_time_in_60s = sampling_window_time_ns / 1e9 * 60 / (window_time_ns / 1e9);
+    double num_this_windows_in_60s = 60 * 1e9 / this_window_time_ns;
+    double real_total_sampling_time_in_60s = sampler->sampling_time_since_last_readjustment_ns * num_this_windows_in_60s / 1e9;
 
-    fprintf(stderr, "[dds.%s] readjusting...\n", sampler->debug_name);
+    const char* readjustment_reason = should_readjust_based_on_time ? "time" : "samples";
+
+    fprintf(stderr, "[dds.%s] readjusting due to %s...\n", sampler->debug_name, readjustment_reason);
+    fprintf(stderr, "events_since_last_readjustment=%ld\n", sampler->events_since_last_readjustment);
     fprintf(stderr, "samples_since_last_readjustment=%ld\n", sampler->samples_since_last_readjustment);
-    fprintf(stderr, "window_time=%ld\n", window_time_ns);
+    fprintf(stderr, "this_window_time=%ld\n", this_window_time_ns);
+    fprintf(stderr, "this_window_sampling_time=%ld\n", sampler->sampling_time_since_last_readjustment_ns);
+    fprintf(stderr, "this_working_window_time=%ld\n", this_window_time_ns - sampler->sampling_time_since_last_readjustment_ns);
+    fprintf(stderr, "this_window_sampling_target_time=%ld\n", this_window_sampling_target_time_ns);
+    fprintf(stderr, "this_window_sampling_overshoot_time=%ld\n", this_window_sampling_overshoot_time_ns);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "target_overhead=%f\n", sampler->target_overhead);
+    fprintf(stderr, "target_overhead_adjustment=%f\n", sampler->target_overhead_adjustment);
     fprintf(stderr, "events_per_sec=%f\n", sampler->events_per_ns * 1e9);
     fprintf(stderr, "sampling_time=%ld\n", sampler->sampling_time_ns);
-    fprintf(stderr, "sampling_window_time=%ld\n", sampling_window_time_ns);
-    fprintf(stderr, "sampling_target_time=%ld\n", reference_target_sampling_time_ns);
-    fprintf(stderr, "sampling_overshoot_time=%ld\n", sampling_overshoot_time_ns);
-    fprintf(stderr, "working_window_time=%ld\n", working_window_time_ns);
+    fprintf(stderr, "avg_sampling_overhead=%f\n", avg_sampling_overhead * 100);
     fprintf(stderr, "sampling_interval=%zu\n", sampler->sampling_interval);
-    fprintf(stderr, "sampling_probability=%f\n", sampler->sampling_probability);
+    fprintf(stderr, "sampling_probability=%f\n", sampler->sampling_probability * 100);
+    fprintf(stderr, "\n");
     fprintf(stderr, "expected allocs in 60s=%f\n", allocs_in_60s);
     fprintf(stderr, "expected samples in 60s=%f\n", samples_in_60s);
     fprintf(stderr, "expected sampling time in 60s=%f (previous real=%f)\n", expected_total_sampling_time_in_60s, real_total_sampling_time_in_60s);
-    fprintf(stderr, "target_overhead=%f\n", sampler->target_overhead);
-    fprintf(stderr, "target_overhead_adjustment=%f\n", sampler->target_overhead_adjustment);
-    fprintf(stderr, "target_sampling_time=%ld\n", target_sampling_time_ns);
     fprintf(stderr, "expected max overhead in 60s=%f\n", target_overhead / 100.0 * 60);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "sampling_time_clamps=%zu\n", sampler->sampling_time_clamps);
     fprintf(stderr, "-------\n");
   #endif
 
@@ -255,15 +290,34 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
   sampler->has_completed_full_adjustment_window = true;
 }
 
+VALUE discrete_dynamic_sampler_state_snapshot(discrete_dynamic_sampler *sampler) {
+  VALUE arguments[] = {
+    ID2SYM(rb_intern("target_overhead")),                 /* => */ DBL2NUM(sampler->target_overhead),
+    ID2SYM(rb_intern("target_overhead_adjustment")),      /* => */ DBL2NUM(sampler->target_overhead_adjustment),
+    ID2SYM(rb_intern("events_per_sec")),                  /* => */ DBL2NUM(sampler->events_per_ns * 1e9),
+    ID2SYM(rb_intern("sampling_time_ns")),                /* => */ LONG2NUM(sampler->sampling_time_ns),
+    ID2SYM(rb_intern("sampling_interval")),               /* => */ ULONG2NUM(sampler->sampling_interval),
+    ID2SYM(rb_intern("sampling_probability")),            /* => */ DBL2NUM(sampler->sampling_probability * 100),
+    ID2SYM(rb_intern("events_since_last_readjustment")),  /* => */ ULONG2NUM(sampler->events_since_last_readjustment),
+    ID2SYM(rb_intern("samples_since_last_readjustment")), /* => */ ULONG2NUM(sampler->samples_since_last_readjustment),
+    ID2SYM(rb_intern("max_sampling_time_ns")),            /* => */ LONG2NUM(sampler->max_sampling_time_ns),
+    ID2SYM(rb_intern("sampling_time_clamps")),            /* => */ ULONG2NUM(sampler->sampling_time_clamps),
+  };
+  VALUE hash = rb_hash_new();
+  for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(hash, arguments[i], arguments[i+1]);
+  return hash;
+}
+
 // ---
 // Below here is boilerplate to expose the above code to Ruby so that we can test it with RSpec as usual.
 
 static VALUE _native_new(VALUE klass);
+static VALUE _native_initialize(VALUE self, VALUE now);
 static VALUE _native_reset(VALUE self, VALUE now);
 static VALUE _native_set_overhead_target_percentage(VALUE self, VALUE target_overhead, VALUE now);
 static VALUE _native_should_sample(VALUE self, VALUE now);
 static VALUE _native_after_sample(VALUE self, VALUE now);
-static VALUE _native_probability(VALUE self);
+static VALUE _native_state_snapshot(VALUE self);
 
 typedef struct sampler_state {
   discrete_dynamic_sampler sampler;
@@ -276,12 +330,15 @@ void collectors_discrete_dynamic_sampler_init(VALUE profiling_module) {
   VALUE sampler_class = rb_define_class_under(testing_module, "Sampler", rb_cObject);
 
   rb_define_alloc_func(sampler_class, _native_new);
+  // NOTE: Despite being native, we're using the normal ruby keyword to prevent having to write a whole
+  //       new ruby file to simply proxy the initialization call.
+  rb_define_method(sampler_class, "initialize", _native_initialize, 1);
 
   rb_define_method(sampler_class, "_native_reset", _native_reset, 1);
   rb_define_method(sampler_class, "_native_set_overhead_target_percentage", _native_set_overhead_target_percentage, 2);
   rb_define_method(sampler_class, "_native_should_sample", _native_should_sample, 1);
   rb_define_method(sampler_class, "_native_after_sample", _native_after_sample, 1);
-  rb_define_method(sampler_class, "_native_probability", _native_probability, 0);
+  rb_define_method(sampler_class, "_native_state_snapshot", _native_state_snapshot, 0);
 }
 
 static const rb_data_type_t sampler_typed_data = {
@@ -296,21 +353,37 @@ static const rb_data_type_t sampler_typed_data = {
 static VALUE _native_new(VALUE klass) {
   sampler_state *state = ruby_xcalloc(sizeof(sampler_state), 1);
 
-  discrete_dynamic_sampler_init(&state->sampler, "test sampler");
+  long now_ns = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
+  if (now_ns == 0) {
+    rb_raise(rb_eRuntimeError, "failed to get clock time");
+  }
+  discrete_dynamic_sampler_init(&state->sampler, "test sampler", now_ns);
 
   return TypedData_Wrap_Struct(klass, &sampler_typed_data, state);
 }
 
-static VALUE _native_reset(VALUE self, VALUE now_ns) {
+static VALUE _native_initialize(VALUE self, VALUE now_ns) {
   ENFORCE_TYPE(now_ns, T_FIXNUM);
 
   sampler_state *state;
   TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
 
-  _discrete_dynamic_sampler_reset(&state->sampler, NUM2LONG(now_ns));
+  discrete_dynamic_sampler_init(&state->sampler, "test sampler", NUM2LONG(now_ns));
+
   return Qtrue;
 }
 
+static VALUE _native_reset(VALUE self, VALUE now_ns) {
+  ENFORCE_TYPE(now_ns, T_FIXNUM);
+
+  sampler_state *state;
+  TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
+
+  discrete_dynamic_sampler_reset(&state->sampler, NUM2LONG(now_ns));
+
+  return Qnil;
+}
+
 static VALUE _native_set_overhead_target_percentage(VALUE self, VALUE target_overhead, VALUE now_ns) {
   ENFORCE_TYPE(target_overhead, T_FLOAT);
   ENFORCE_TYPE(now_ns, T_FIXNUM);
@@ -318,7 +391,7 @@ static VALUE _native_set_overhead_target_percentage(VALUE self, VALUE target_ove
   sampler_state *state;
   TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
 
-  _discrete_dynamic_sampler_set_overhead_target_percentage(&state->sampler, NUM2DBL(target_overhead), NUM2LONG(now_ns));
+  discrete_dynamic_sampler_set_overhead_target_percentage(&state->sampler, NUM2DBL(target_overhead), NUM2LONG(now_ns));
 
   return Qnil;
 }
@@ -329,7 +402,7 @@ VALUE _native_should_sample(VALUE self, VALUE now_ns) {
   sampler_state *state;
   TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
 
-  return _discrete_dynamic_sampler_should_sample(&state->sampler, NUM2LONG(now_ns)) ? Qtrue : Qfalse;
+  return discrete_dynamic_sampler_should_sample(&state->sampler, NUM2LONG(now_ns)) ? Qtrue : Qfalse;
 }
 
 VALUE _native_after_sample(VALUE self, VALUE now_ns) {
@@ -338,12 +411,12 @@ VALUE _native_after_sample(VALUE self, VALUE now_ns) {
   sampler_state *state;
   TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
 
-  return LONG2NUM(_discrete_dynamic_sampler_after_sample(&state->sampler, NUM2LONG(now_ns)));
+  return LONG2NUM(discrete_dynamic_sampler_after_sample(&state->sampler, NUM2LONG(now_ns)));
 }
 
-VALUE _native_probability(VALUE self) {
+VALUE _native_state_snapshot(VALUE self) {
   sampler_state *state;
   TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
 
-  return DBL2NUM(discrete_dynamic_sampler_probability(&state->sampler));
+  return discrete_dynamic_sampler_state_snapshot(&state->sampler);
 }

data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.h

@@ -3,6 +3,8 @@
 #include <stdbool.h>
 #include <stddef.h>
 
+#include <ruby.h>
+
 // A sampler that will sample discrete events based on the overhead of their
 // sampling.
 //
@@ -31,6 +33,8 @@ typedef struct discrete_dynamic_sampler {
   // NOTE: This is an inverted view of the probability.
   // NOTE: A value of 0 works as +inf, effectively disabling sampling (to align with probability=0)
   unsigned long sampling_interval;
+  // Max allowed value for an individual sampling time measurement.
+  long max_sampling_time_ns;
 
   // -- Sampling State --
   // How many events have we seen since we last decided to sample.
@@ -53,20 +57,23 @@ typedef struct discrete_dynamic_sampler {
   // A negative number that we add to target_overhead to serve as extra padding to
   // try and mitigate observed overshooting of max sampling time.
   double target_overhead_adjustment;
+
+  // -- Interesting stats --
+  unsigned long sampling_time_clamps;
 } discrete_dynamic_sampler;
 
 
 // Init a new sampler with sane defaults.
-void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name);
+void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name, long now_ns);
 
 // Reset a sampler, clearing all stored state.
-void discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler);
+void discrete_dynamic_sampler_reset(discrete_dynamic_sampler *sampler, long now_ns);
 
 // Sets a new target_overhead for the provided sampler, resetting it in the process.
 // @param target_overhead A double representing the percentage of total time we are
 //        willing to use as overhead for the resulting sampling. Values are expected
 //        to be in the range ]0.0, 100.0].
-void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead);
+void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sampler *sampler, double target_overhead, long now_ns);
 
 // Make a sampling decision.
 //
@@ -75,15 +82,20 @@ void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sa
 //
 // NOTE: If true is returned we implicitly assume the start of a sampling operation
 //       and it is expected that a follow-up after_sample call is issued.
-bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler);
+bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler, long now_ns);
 
 // Signal the end of a sampling operation.
 //
 // @return Sampling time in nanoseconds for the sample operation we just finished.
-long discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler);
+long discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler, long now_ns);
 
 // Retrieve the current sampling probability ([0.0, 100.0]) being applied by this sampler.
 double discrete_dynamic_sampler_probability(discrete_dynamic_sampler *sampler);
 
 // Retrieve the current number of events seen since last sample.
 unsigned long discrete_dynamic_sampler_events_since_last_sample(discrete_dynamic_sampler *sampler);
+
+// Return a Ruby hash containing a snapshot of this sampler's interesting state at calling time.
+// WARN: This allocates in the Ruby VM and therefore should not be called without the
+//       VM lock or during GC.
+VALUE discrete_dynamic_sampler_state_snapshot(discrete_dynamic_sampler *sampler);

data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_thread_context.c

@@ -82,6 +82,9 @@ static ID at_id_id;           // id of :@id in Ruby
 static ID at_resource_id;     // id of :@resource in Ruby
 static ID at_root_span_id;    // id of :@root_span in Ruby
 static ID at_type_id;         // id of :@type in Ruby
+static ID at_otel_values_id;  // id of :@otel_values in Ruby
+static ID at_parent_span_id_id; // id of :@parent_span_id in Ruby
+static ID at_datadog_trace_id;  // id of :@datadog_trace in Ruby
 
 // Contains state for a single ThreadContext instance
 struct thread_context_collector_state {
@@ -114,6 +117,8 @@ struct thread_context_collector_state {
   monotonic_to_system_epoch_state time_converter_state;
   // Used to identify the main thread, to give it a fallback name
   VALUE main_thread;
+  // Used when extracting trace identifiers from otel spans. Lazily initialized.
+  VALUE otel_current_span_key;
 
   struct stats {
     // Track how many garbage collection samples we've taken.
@@ -218,6 +223,14 @@ static VALUE thread_list(struct thread_context_collector_state *state);
 static VALUE _native_sample_allocation(DDTRACE_UNUSED VALUE self, VALUE collector_instance, VALUE sample_weight, VALUE new_object);
 static VALUE _native_new_empty_thread(VALUE self);
 static ddog_CharSlice ruby_value_type_to_class_name(enum ruby_value_type type);
+static void ddtrace_otel_trace_identifiers_for(
+  struct thread_context_collector_state *state,
+  VALUE *active_trace,
+  VALUE *root_span,
+  VALUE *numeric_span_id,
+  VALUE active_span,
+  VALUE otel_values
+);
 
 void collectors_thread_context_init(VALUE profiling_module) {
   VALUE collectors_module = rb_define_module_under(profiling_module, "Collectors");
@@ -255,6 +268,9 @@ void collectors_thread_context_init(VALUE profiling_module) {
   at_resource_id = rb_intern_const("@resource");
   at_root_span_id = rb_intern_const("@root_span");
   at_type_id = rb_intern_const("@type");
+  at_otel_values_id = rb_intern_const("@otel_values");
+  at_parent_span_id_id = rb_intern_const("@parent_span_id");
+  at_datadog_trace_id = rb_intern_const("@datadog_trace");
 
   gc_profiling_init();
 }
@@ -282,6 +298,7 @@ static void thread_context_collector_typed_data_mark(void *state_ptr) {
   st_foreach(state->hash_map_per_thread_context, hash_map_per_thread_context_mark, 0 /* unused */);
   rb_gc_mark(state->thread_list_buffer);
   rb_gc_mark(state->main_thread);
+  rb_gc_mark(state->otel_current_span_key);
 }
 
 static void thread_context_collector_typed_data_free(void *state_ptr) {
@@ -334,6 +351,7 @@ static VALUE _native_new(VALUE klass) {
   state->allocation_type_enabled = true;
   state->time_converter_state = (monotonic_to_system_epoch_state) MONOTONIC_TO_SYSTEM_EPOCH_INITIALIZER;
   state->main_thread = rb_thread_main();
+  state->otel_current_span_key = Qnil;
   state->gc_tracking.wall_time_at_previous_gc_ns = INVALID_TIME;
   state->gc_tracking.wall_time_at_last_flushed_gc_event_ns = 0;
 
@@ -603,11 +621,14 @@ bool thread_context_collector_on_gc_finish(VALUE self_instance) {
   // Let the caller know if it should schedule a flush or not. Returning true every time would cause a lot of overhead
   // on the application (see GC tracking introduction at the top of the file), so instead we try to accumulate a few
   // samples first.
-  bool finished_major_gc = gc_profiling_has_major_gc_finished();
   bool over_flush_time_treshold =
     (wall_time_at_finish_ns - state->gc_tracking.wall_time_at_last_flushed_gc_event_ns) >= TIME_BETWEEN_GC_EVENTS_NS;
 
-  return finished_major_gc || over_flush_time_treshold;
+  if (over_flush_time_treshold) {
+    return true;
+  } else {
+    return gc_profiling_has_major_gc_finished();
+  }
 }
 
 // This function gets called after one or more GC work steps (calls to on_gc_start/on_gc_finish).
@@ -917,6 +938,7 @@ static VALUE _native_inspect(DDTRACE_UNUSED VALUE _self, VALUE collector_instanc
   ));
   rb_str_concat(result, rb_sprintf(" main_thread=%"PRIsVALUE, state->main_thread));
   rb_str_concat(result, rb_sprintf(" gc_tracking=%"PRIsVALUE, gc_tracking_as_ruby_hash(state)));
+  rb_str_concat(result, rb_sprintf(" otel_current_span_key=%"PRIsVALUE, state->otel_current_span_key));
 
   return result;
 }
@@ -1104,10 +1126,19 @@ static void trace_identifiers_for(struct thread_context_collector_state *state,
 
   VALUE root_span = rb_ivar_get(active_trace, at_root_span_id /* @root_span */);
   VALUE active_span = rb_ivar_get(active_trace, at_active_span_id /* @active_span */);
-  if (root_span == Qnil || active_span == Qnil) return;
+  // Note: On Ruby 3.x `rb_attr_get` is exactly the same as `rb_ivar_get`. For Ruby 2.x, the difference is that
+  // `rb_ivar_get` can trigger "warning: instance variable @otel_values not initialized" if warnings are enabled and
+  // opentelemetry is not in use, whereas `rb_attr_get` does the lookup without generating the warning.
+  VALUE otel_values = rb_attr_get(active_trace, at_otel_values_id /* @otel_values */);
+
+  VALUE numeric_span_id = Qnil;
+
+  if (otel_values != Qnil) ddtrace_otel_trace_identifiers_for(state, &active_trace, &root_span, &numeric_span_id, active_span, otel_values);
+
+  if (root_span == Qnil || (active_span == Qnil && numeric_span_id == Qnil)) return;
 
   VALUE numeric_local_root_span_id = rb_ivar_get(root_span, at_id_id /* @id */);
-  VALUE numeric_span_id = rb_ivar_get(active_span, at_id_id /* @id */);
+  if (active_span != Qnil && numeric_span_id == Qnil) numeric_span_id = rb_ivar_get(active_span, at_id_id /* @id */);
   if (numeric_local_root_span_id == Qnil || numeric_span_id == Qnil) return;
 
   trace_identifiers_result->local_root_span_id = NUM2ULL(numeric_local_root_span_id);
@@ -1299,3 +1330,65 @@ static ddog_CharSlice ruby_value_type_to_class_name(enum ruby_value_type type) {
                   default: return DDOG_CHARSLICE_C("(VM Internal, Missing class)");
   }
 }
+
+static VALUE get_otel_current_span_key(struct thread_context_collector_state *state) {
+  if (state->otel_current_span_key == Qnil) {
+    VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+    VALUE opentelemetry_module = rb_const_get(datadog_module, rb_intern("OpenTelemetry"));
+    VALUE api_module = rb_const_get(opentelemetry_module, rb_intern("API"));
+    VALUE context_module = rb_const_get(api_module, rb_intern_const("Context"));
+    VALUE current_span_key = rb_const_get(context_module, rb_intern_const("CURRENT_SPAN_KEY"));
+
+    if (current_span_key == Qnil) {
+      rb_raise(rb_eRuntimeError, "Unexpected: Missing Datadog::OpenTelemetry::API::Context::CURRENT_SPAN_KEY");
+    }
+
+    state->otel_current_span_key = current_span_key;
+  }
+
+  return state->otel_current_span_key;
+}
+
+// This method gets used when ddtrace is being used indirectly via the otel APIs. Information gets stored slightly
+// differently, and this codepath handles it.
+static void ddtrace_otel_trace_identifiers_for(
+  struct thread_context_collector_state *state,
+  VALUE *active_trace,
+  VALUE *root_span,
+  VALUE *numeric_span_id,
+  VALUE active_span,
+  VALUE otel_values
+) {
+  VALUE resolved_numeric_span_id =
+    active_span == Qnil ?
+      // For traces started from otel spans, the span id will be empty, and the @parent_span_id has the right value
+      rb_ivar_get(*active_trace, at_parent_span_id_id /* @parent_span_id */) :
+      // Regular span created by ddtrace
+      rb_ivar_get(active_span, at_id_id /* @id */);
+
+  if (resolved_numeric_span_id == Qnil) return;
+
+  VALUE otel_current_span_key = get_otel_current_span_key(state);
+  VALUE current_trace = *active_trace;
+
+  // ddtrace uses a different structure when spans are created from otel, where each otel span will have a unique ddtrace
+  // trace and span representing it. Each ddtrace trace is then connected to the previous otel span, forming a linked
+  // list. The local root span is going to be the trace/span we find at the end of this linked list.
+  while (otel_values != Qnil) {
+    VALUE otel_span = rb_hash_lookup(otel_values, otel_current_span_key);
+    if (otel_span == Qnil) break;
+    VALUE next_trace = rb_ivar_get(otel_span, at_datadog_trace_id);
+    if (next_trace == Qnil) break;
+
+    current_trace = next_trace;
+    otel_values = rb_ivar_get(current_trace, at_otel_values_id /* @otel_values */);
+  }
+
+  // We found the last trace in the linked list. This contains the local root span
+  VALUE resolved_root_span = rb_ivar_get(current_trace, at_root_span_id /* @root_span */);
+  if (resolved_root_span == Qnil) return;
+
+  *root_span = resolved_root_span;
+  *active_trace = current_trace;
+  *numeric_span_id = resolved_numeric_span_id;
+}