datadog 2.4.0 → 2.5.0

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -256,21 +256,21 @@ if Datadog::Profiling::NativeExtensionHelpers::CAN_USE_MJIT_HEADER
   create_makefile EXTENSION_NAME
 else
   # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  # the debase-ruby_core_source gem to get access to private VM headers.
+  # the datadog-ruby_core_source gem to get access to private VM headers.
   # This gem ships source code copies of these VM headers for the different Ruby VM versions;
-  # see https://github.com/ruby-debug/debase-ruby_core_source for details
+  # see https://github.com/DataDog/datadog-ruby_core_source for details
 
   create_header
 
-  require "debase/ruby_core_source"
+  require "datadog/ruby_core_source"
   dir_config("ruby") # allow user to pass in non-standard core include directory
 
   # This is a workaround for a weird issue...
   #
-  # The mkmf tool defines a `with_cppflags` helper that debase-ruby_core_source uses. This helper temporarily
+  # The mkmf tool defines a `with_cppflags` helper that datadog-ruby_core_source uses. This helper temporarily
   # replaces `$CPPFLAGS` (aka the C pre-processor [not c++!] flags) with a different set when doing something.
   #
-  # The debase-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
+  # The datadog-ruby_core_source gem uses `with_cppflags` during makefile generation to inject extra headers into the
   # path. But because `with_cppflags` replaces `$CPPFLAGS`, well, the default `$CPPFLAGS` are not included in the
   # makefile.
   #
@@ -281,12 +281,12 @@ else
   # `VM_CHECK_MODE=1` when building Ruby will trigger this issue (because somethings in structures the profiler reads
   # are ifdef'd out using this setting).
   #
-  # To workaround this issue, we override `with_cppflags` for debase-ruby_core_source to still include `$CPPFLAGS`.
-  Debase::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+  # To workaround this issue, we override `with_cppflags` for datadog-ruby_core_source to still include `$CPPFLAGS`.
+  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
     super("#{newflags} #{$CPPFLAGS}", &block)
   end
 
-  Debase::RubyCoreSource
+  Datadog::RubyCoreSource
     .create_makefile_with_core(
       proc do
         headers_available =

data/ext/datadog_profiling_native_extension/heap_recorder.c

@@ -5,6 +5,7 @@
 #include <errno.h>
 #include "collectors_stack.h"
 #include "libdatadog_helpers.h"
+#include "time_helpers.h"
 
 #if (defined(HAVE_WORKING_RB_GC_FORCE_RECYCLE) && ! defined(NO_SEEN_OBJ_ID_FLAG))
   #define CAN_APPLY_GC_FORCE_RECYCLE_BUG_WORKAROUND
@@ -16,6 +17,16 @@
 // relevant for heap profiles as the great majority should be trivially reclaimed
 // during the next GC.
 #define ITERATION_MIN_AGE 1
+// Copied from https://github.com/ruby/ruby/blob/15135030e5808d527325feaaaf04caeb1b44f8b5/gc/default.c#L725C1-L725C27
+// to align with Ruby's GC definition of what constitutes an old object which are only
+// supposed to be reclaimed in major GCs.
+#define OLD_AGE 3
+// Wait at least 2 seconds before asking heap recorder to explicitly update itself. Heap recorder
+// data will only materialize at profile serialization time but updating often helps keep our
+// heap tracking data small since every GC should get rid of a bunch of temporary objects. The
+// more we clean up before profile flush, the less work we'll have to do all-at-once when preparing
+// to flush heap data and holding the GVL which should hopefully help with reducing latency impact.
+#define MIN_TIME_BETWEEN_HEAP_RECORDER_UPDATES_NS SECONDS_AS_NS(2)
 
 // A compact representation of a stacktrace frame for a heap allocation.
 typedef struct {
@@ -144,11 +155,18 @@ struct heap_recorder {
   // mutation of the data so iteration can occur without acquiring a lock.
   // NOTE: Contrary to object_records, this table has no ownership of its data.
   st_table *object_records_snapshot;
-  // The GC gen/epoch/count in which we prepared the current iteration.
+  // Are we currently updating or not?
+  bool updating;
+  // The GC gen/epoch/count in which we are updating (or last updated if not currently updating).
   //
-  // This enables us to calculate the age of iterated objects in the above snapshot by
-  // comparing it against an object's alloc_gen.
-  size_t iteration_gen;
+  // This enables us to calculate the age of objects considered in the update by comparing it
+  // against an object's alloc_gen.
+  size_t update_gen;
+  // Whether the current update (or last update if not currently updating) is including old
+  // objects or not.
+  bool update_include_old;
+  // When did we do the last update of heap recorder?
+  long last_update_ns;
 
   // Data for a heap recording that was started but not yet ended
   recording active_recording;
@@ -165,6 +183,21 @@ struct heap_recorder {
     size_t objects_skipped;
     size_t objects_frozen;
   } stats_last_update;
+
+  struct stats_lifetime {
+    unsigned long updates_successful;
+    unsigned long updates_skipped_concurrent;
+    unsigned long updates_skipped_gcgen;
+    unsigned long updates_skipped_time;
+
+    double ewma_young_objects_alive;
+    double ewma_young_objects_dead;
+    double ewma_young_objects_skipped; // Note: Here "young" refers to the young update; objects skipped includes non-young objects
+
+    double ewma_objects_alive;
+    double ewma_objects_dead;
+    double ewma_objects_skipped;
+  } stats_lifetime;
 };
 
 struct end_heap_allocation_args {
@@ -183,6 +216,8 @@ static int st_object_records_debug(st_data_t key, st_data_t value, st_data_t ext
 static int update_object_record_entry(st_data_t*, st_data_t*, st_data_t, int);
 static void commit_recording(heap_recorder*, heap_record*, recording);
 static VALUE end_heap_allocation_recording(VALUE end_heap_allocation_args);
+static void heap_recorder_update(heap_recorder *heap_recorder, bool full_update);
+static inline double ewma_stat(double previous, double current);
 
 // ==========================
 // Heap Recorder External API
@@ -280,6 +315,9 @@ void heap_recorder_after_fork(heap_recorder *heap_recorder) {
   if (heap_recorder->object_records_snapshot != NULL) {
     heap_recorder_finish_iteration(heap_recorder);
   }
+
+  // Clear lifetime stats since this is essentially a new heap recorder
+  heap_recorder->stats_lifetime = (struct stats_lifetime) {0};
 }
 
 void start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj, unsigned int weight, ddog_CharSlice *alloc_class) {
@@ -394,23 +432,94 @@ static VALUE end_heap_allocation_recording(VALUE end_heap_allocation_args) {
   return Qnil;
 }
 
-void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
+void heap_recorder_update_young_objects(heap_recorder *heap_recorder) {
   if (heap_recorder == NULL) {
     return;
   }
 
-  heap_recorder->iteration_gen = rb_gc_count();
+  heap_recorder_update(heap_recorder, /* full_update: */ false);
+}
+
+static void heap_recorder_update(heap_recorder *heap_recorder, bool full_update) {
+  if (heap_recorder->updating) {
+    if (full_update) rb_raise(rb_eRuntimeError, "BUG: full_update should not be triggered during another update");
+
+    // If we try to update while another update is still running, short-circuit.
+    // NOTE: This runs while holding the GVL. But since updates may be triggered from GC activity, there's still
+    //       a chance for updates to be attempted concurrently if scheduling gods so determine.
+    heap_recorder->stats_lifetime.updates_skipped_concurrent++;
+    return;
+  }
 
   if (heap_recorder->object_records_snapshot != NULL) {
-    // we could trivially handle this but we raise to highlight and catch unexpected usages.
-    rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
+    // While serialization is happening, it runs without the GVL and uses the object_records_snapshot.
+    // Although we iterate on a snapshot of object_records, these records point to other data that has not been
+    // snapshotted for efficiency reasons (e.g. heap_records). Since updating may invalidate
+    // some of that non-snapshotted data, let's refrain from doing updates during iteration. This also enforces the
+    // semantic that iteration will operate as a point-in-time snapshot.
+    return;
   }
 
+  size_t current_gc_gen = rb_gc_count();
+  long now_ns = monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE);
+
+  if (!full_update) {
+    if (current_gc_gen == heap_recorder->update_gen) {
+      // Are we still in the same GC gen as last update? If so, skip updating since things should not have
+      // changed significantly since last time.
+      // NOTE: This is mostly a performance decision. I suppose some objects may be cleaned up in intermediate
+      // GC steps and sizes may change. But because we have to iterate through all our tracked
+      // object records to do an update, let's wait until all steps for a particular GC generation
+      // have finished to do so. We may revisit this once we have a better liveness checking mechanism.
+      heap_recorder->stats_lifetime.updates_skipped_gcgen++;
+      return;
+    }
+
+    if (now_ns > 0 && (now_ns - heap_recorder->last_update_ns) < MIN_TIME_BETWEEN_HEAP_RECORDER_UPDATES_NS) {
+      // We did an update not too long ago. Let's skip this one to avoid over-taxing the system.
+      heap_recorder->stats_lifetime.updates_skipped_time++;
+      return;
+    }
+  }
+
+  heap_recorder->updating = true;
   // Reset last update stats, we'll be building them from scratch during the st_foreach call below
-  heap_recorder->stats_last_update = (struct stats_last_update) {};
+  heap_recorder->stats_last_update = (struct stats_last_update) {0};
+
+  heap_recorder->update_gen = current_gc_gen;
+  heap_recorder->update_include_old = full_update;
 
   st_foreach(heap_recorder->object_records, st_object_record_update, (st_data_t) heap_recorder);
 
+  heap_recorder->last_update_ns = now_ns;
+  heap_recorder->stats_lifetime.updates_successful++;
+
+  // Lifetime stats updating
+  if (!full_update) {
+    heap_recorder->stats_lifetime.ewma_young_objects_alive = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_alive, heap_recorder->stats_last_update.objects_alive);
+    heap_recorder->stats_lifetime.ewma_young_objects_dead = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_dead, heap_recorder->stats_last_update.objects_dead);
+    heap_recorder->stats_lifetime.ewma_young_objects_skipped = ewma_stat(heap_recorder->stats_lifetime.ewma_young_objects_skipped, heap_recorder->stats_last_update.objects_skipped);
+  } else {
+    heap_recorder->stats_lifetime.ewma_objects_alive = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_alive, heap_recorder->stats_last_update.objects_alive);
+    heap_recorder->stats_lifetime.ewma_objects_dead = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_dead, heap_recorder->stats_last_update.objects_dead);
+    heap_recorder->stats_lifetime.ewma_objects_skipped = ewma_stat(heap_recorder->stats_lifetime.ewma_objects_skipped, heap_recorder->stats_last_update.objects_skipped);
+  }
+
+  heap_recorder->updating = false;
+}
+
+void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
+  if (heap_recorder == NULL) {
+    return;
+  }
+
+  if (heap_recorder->object_records_snapshot != NULL) {
+    // we could trivially handle this but we raise to highlight and catch unexpected usages.
+    rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
+  }
+
+  heap_recorder_update(heap_recorder, /* full_update: */ true);
+
   heap_recorder->object_records_snapshot = st_copy(heap_recorder->object_records);
   if (heap_recorder->object_records_snapshot == NULL) {
     rb_raise(rb_eRuntimeError, "Failed to create heap snapshot.");
@@ -474,6 +583,19 @@ VALUE heap_recorder_state_snapshot(heap_recorder *heap_recorder) {
     ID2SYM(rb_intern("last_update_objects_dead")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_dead),
     ID2SYM(rb_intern("last_update_objects_skipped")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_skipped),
     ID2SYM(rb_intern("last_update_objects_frozen")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_frozen),
+
+    // Lifetime stats
+    ID2SYM(rb_intern("lifetime_updates_successful")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_successful),
+    ID2SYM(rb_intern("lifetime_updates_skipped_concurrent")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_concurrent),
+    ID2SYM(rb_intern("lifetime_updates_skipped_gcgen")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_gcgen),
+    ID2SYM(rb_intern("lifetime_updates_skipped_time")), /* => */ LONG2NUM(heap_recorder->stats_lifetime.updates_skipped_time),
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_alive")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_alive),
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_dead")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_dead),
+      // Note: Here "young" refers to the young update; objects skipped includes non-young objects
+    ID2SYM(rb_intern("lifetime_ewma_young_objects_skipped")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_young_objects_skipped),
+    ID2SYM(rb_intern("lifetime_ewma_objects_alive")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_alive),
+    ID2SYM(rb_intern("lifetime_ewma_objects_dead")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_dead),
+    ID2SYM(rb_intern("lifetime_ewma_objects_skipped")), /* => */ DBL2NUM(heap_recorder->stats_lifetime.ewma_objects_skipped),
   };
   VALUE hash = rb_hash_new();
   for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(hash, arguments[i], arguments[i+1]);
@@ -503,11 +625,14 @@ void heap_recorder_testonly_assert_hash_matches(ddog_prof_Slice_Location locatio
 
 VALUE heap_recorder_testonly_debug(heap_recorder *heap_recorder) {
   if (heap_recorder == NULL) {
-    return rb_str_new2("NULL heap_recorder");
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
   }
 
   VALUE debug_str = rb_str_new2("object records:\n");
   st_foreach(heap_recorder->object_records, st_object_records_debug, (st_data_t) debug_str);
+
+  rb_str_catf(debug_str, "state snapshot: %"PRIsVALUE"\n------\n", heap_recorder_state_snapshot(heap_recorder));
+
   return debug_str;
 }
 
@@ -526,13 +651,6 @@ static int st_object_record_entry_free(DDTRACE_UNUSED st_data_t key, st_data_t v
   return ST_DELETE;
 }
 
-// Check to see if an object should not be included in a heap recorder iteration.
-// This centralizes the checking logic to ensure it's equally applied between
-// preparation and iteration codepaths.
-static inline bool should_exclude_from_iteration(object_record *obj_record) {
-  return obj_record->object_data.gen_age < ITERATION_MIN_AGE;
-}
-
 static int st_object_record_update(st_data_t key, st_data_t value, st_data_t extra_arg) {
   long obj_id = (long) key;
   object_record *record = (object_record*) value;
@@ -540,16 +658,20 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
 
   VALUE ref;
 
-  size_t iteration_gen = recorder->iteration_gen;
+  size_t update_gen = recorder->update_gen;
   size_t alloc_gen = record->object_data.alloc_gen;
   // Guard against potential overflows given unsigned types here.
-  record->object_data.gen_age = alloc_gen < iteration_gen ? iteration_gen - alloc_gen : 0;
+  record->object_data.gen_age = alloc_gen < update_gen ? update_gen - alloc_gen : 0;
+
+  if (record->object_data.gen_age == 0) {
+    // Objects that belong to the current GC gen have not had a chance to be cleaned up yet
+    // and won't show up in the iteration anyway so no point in checking their liveness/sizes.
+    recorder->stats_last_update.objects_skipped++;
+    return ST_CONTINUE;
+  }
 
-  if (should_exclude_from_iteration(record)) {
-    // If an object won't be included in the current iteration, there's
-    // no point checking for liveness or updating its size, so exit early.
-    // NOTE: This means that there should be an equivalent check during actual
-    //       iteration otherwise we'd iterate/expose stale object data.
+  if (!recorder->update_include_old && record->object_data.gen_age >= OLD_AGE) {
+    // The current update is not including old objects but this record is for an old object, skip its update.
     recorder->stats_last_update.objects_skipped++;
     return ST_CONTINUE;
   }
@@ -598,7 +720,11 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
 
   #endif
 
-  if (recorder->size_enabled && !record->object_data.is_frozen) {
+  if (
+    recorder->size_enabled &&
+    recorder->update_include_old && // We only update sizes when doing a full update
+    !record->object_data.is_frozen
+  ) {
     // if we were asked to update sizes and this object was not already seen as being frozen,
     // update size again.
     record->object_data.size = ruby_obj_memsize_of(ref);
@@ -622,10 +748,8 @@ static int st_object_records_iterate(DDTRACE_UNUSED st_data_t key, st_data_t val
 
   const heap_recorder *recorder = context->heap_recorder;
 
-  if (should_exclude_from_iteration(record)) {
+  if (record->object_data.gen_age < ITERATION_MIN_AGE) {
     // Skip objects that should not be included in iteration
-    // NOTE: This matches the short-circuiting condition in st_object_record_update
-    //       and prevents iteration over stale objects.
     return ST_CONTINUE;
   }
 
@@ -1087,3 +1211,25 @@ st_index_t heap_record_key_hash_st(st_data_t key) {
     return ddog_location_slice_hash(*record_key->location_slice, FNV1_32A_INIT);
   }
 }
+
+static inline double ewma_stat(double previous, double current) {
+  double alpha = 0.3;
+  return (1 - alpha) * previous + alpha * current;
+}
+
+VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, VALUE obj_id) {
+  if (heap_recorder == NULL) {
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
+  }
+
+  // Check if object records contains an object with this object_id
+  return st_is_member(heap_recorder->object_records, FIX2LONG(obj_id)) ? Qtrue : Qfalse;
+}
+
+void heap_recorder_testonly_reset_last_update(heap_recorder *heap_recorder) {
+  if (heap_recorder == NULL) {
+    rb_raise(rb_eArgError, "heap_recorder is NULL");
+  }
+
+  heap_recorder->last_update_ns = 0;
+}

data/ext/datadog_profiling_native_extension/heap_recorder.h

@@ -118,6 +118,11 @@ void start_heap_allocation_recording(heap_recorder *heap_recorder, VALUE new_obj
 __attribute__((warn_unused_result))
 int end_heap_allocation_recording_with_rb_protect(heap_recorder *heap_recorder, ddog_prof_Slice_Location locations);
 
+// Update the heap recorder, **checking young objects only**. The idea here is to align with GC: most young objects never
+// survive enough GC generations, and thus periodically running this method reduces memory usage (we get rid of
+// these objects quicker) and hopefully reduces tail latency (because there's less objects at serialization time to check).
+void heap_recorder_update_young_objects(heap_recorder *heap_recorder);
+
 // Update the heap recorder to reflect the latest state of the VM and prepare internal structures
 // for efficient iteration.
 //
@@ -166,3 +171,9 @@ void heap_recorder_testonly_assert_hash_matches(ddog_prof_Slice_Location locatio
 // Returns a Ruby string with a representation of internal data helpful to
 // troubleshoot issues such as unexpected test failures.
 VALUE heap_recorder_testonly_debug(heap_recorder *heap_recorder);
+
+// Check if a given object_id is being tracked or not
+VALUE heap_recorder_testonly_is_object_recorded(heap_recorder *heap_recorder, VALUE obj_id);
+
+// Used to ensure that a GC actually triggers an update of the objects
+void heap_recorder_testonly_reset_last_update(heap_recorder *heap_recorder);

data/ext/datadog_profiling_native_extension/native_extension_helpers.rb

@@ -9,7 +9,7 @@ module Datadog
       # Can be set to force rubygems to fail gem installation when profiling extension could not be built
       ENV_FAIL_INSTALL_IF_MISSING_EXTENSION = "DD_PROFILING_FAIL_INSTALL_IF_MISSING_EXTENSION"
 
-      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
+      # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on datadog-ruby_core_source
       CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
 
       def self.fail_install_if_missing_extension?

data/ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -13,7 +13,7 @@
   #include RUBY_MJIT_HEADER
 #else
   // The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
-  // the debase-ruby_core_source gem to get access to private VM headers.
+  // the datadog-ruby_core_source gem to get access to private VM headers.
 
   // We can't do anything about warnings in VM headers, so we just use this technique to suppress them.
   // See https://nelkinda.com/blog/suppress-warnings-in-gcc-and-clang/#d11e364 for details.

data/ext/datadog_profiling_native_extension/ruby_helpers.c

@@ -219,16 +219,19 @@ static bool ruby_is_obj_with_class(VALUE obj) {
   return false;
 }
 
-VALUE ruby_safe_inspect(VALUE obj) {
-  if (!ruby_is_obj_with_class(obj)) {
-    return rb_str_new_cstr("(Not an object)");
-  }
+// These two functions are not present in the VM headers, but are public symbols that can be invoked.
+int rb_objspace_internal_object_p(VALUE obj);
+const char *rb_obj_info(VALUE obj);
 
-  if (rb_respond_to(obj, inspect_id)) {
-    return rb_sprintf("%+"PRIsVALUE, obj);
-  } else if (rb_respond_to(obj, to_s_id)) {
-    return rb_sprintf("%"PRIsVALUE, obj);
-  } else {
-    return rb_str_new_cstr("(Not inspectable)");
-  }
+VALUE ruby_safe_inspect(VALUE obj) {
+  if (!ruby_is_obj_with_class(obj))       return rb_str_new_cstr("(Not an object)");
+  if (rb_objspace_internal_object_p(obj)) return rb_sprintf("(VM Internal, %s)", rb_obj_info(obj));
+  // @ivoanjo: I saw crashes on Ruby 3.1.4 when trying to #inspect matchdata objects. I'm not entirely sure why this
+  // is needed, but since we only use this method for debug purposes I put in this alternative and decided not to
+  // dig deeper.
+  if (rb_type(obj) == RUBY_T_MATCH)   return rb_sprintf("(MatchData, %s)", rb_obj_info(obj));
+  if (rb_respond_to(obj, inspect_id)) return rb_sprintf("%+"PRIsVALUE, obj);
+  if (rb_respond_to(obj, to_s_id))    return rb_sprintf("%"PRIsVALUE, obj);
+
+  return rb_str_new_cstr("(Not inspectable)");
 }

data/ext/datadog_profiling_native_extension/stack_recorder.c

@@ -187,6 +187,7 @@ typedef struct profile_slot {
 struct stack_recorder_state {
   // Heap recorder instance
   heap_recorder *heap_recorder;
+  bool heap_clean_after_gc_enabled;
 
   pthread_mutex_t mutex_slot_one;
   profile_slot profile_slot_one;
@@ -236,16 +237,7 @@ static VALUE _native_new(VALUE klass);
 static void initialize_slot_concurrency_control(struct stack_recorder_state *state);
 static void initialize_profiles(struct stack_recorder_state *state, ddog_prof_Slice_ValueType sample_types);
 static void stack_recorder_typed_data_free(void *data);
-static VALUE _native_initialize(
-  DDTRACE_UNUSED VALUE _self,
-  VALUE recorder_instance,
-  VALUE cpu_time_enabled,
-  VALUE alloc_samples_enabled,
-  VALUE heap_samples_enabled,
-  VALUE heap_size_enabled,
-  VALUE heap_sample_every,
-  VALUE timeline_enabled
-);
+static VALUE _native_initialize(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self);
 static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
 static VALUE ruby_time_from(ddog_Timespec ddprof_time);
 static void *call_serialize_without_gvl(void *call_args);
@@ -270,7 +262,9 @@ static VALUE _native_gc_force_recycle(DDTRACE_UNUSED VALUE _self, VALUE obj);
 static VALUE _native_has_seen_id_flag(DDTRACE_UNUSED VALUE _self, VALUE obj);
 static VALUE _native_stats(DDTRACE_UNUSED VALUE self, VALUE instance);
 static VALUE build_profile_stats(profile_slot *slot, long serialization_time_ns, long heap_iteration_prep_time_ns, long heap_profile_build_time_ns);
-
+static VALUE _native_is_object_recorded(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance, VALUE object_id);
+static VALUE _native_heap_recorder_reset_last_update(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
+static VALUE _native_recorder_after_gc_step(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
 
 void stack_recorder_init(VALUE profiling_module) {
   VALUE stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
@@ -287,7 +281,7 @@ void stack_recorder_init(VALUE profiling_module) {
   // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
   rb_define_alloc_func(stack_recorder_class, _native_new);
 
-  rb_define_singleton_method(stack_recorder_class, "_native_initialize", _native_initialize, 7);
+  rb_define_singleton_method(stack_recorder_class, "_native_initialize", _native_initialize, -1);
   rb_define_singleton_method(stack_recorder_class, "_native_serialize",  _native_serialize, 1);
   rb_define_singleton_method(stack_recorder_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
   rb_define_singleton_method(stack_recorder_class, "_native_stats", _native_stats, 1);
@@ -307,6 +301,9 @@ void stack_recorder_init(VALUE profiling_module) {
       _native_gc_force_recycle, 1);
   rb_define_singleton_method(testing_module, "_native_has_seen_id_flag",
       _native_has_seen_id_flag, 1);
+  rb_define_singleton_method(testing_module, "_native_is_object_recorded?", _native_is_object_recorded, 2);
+  rb_define_singleton_method(testing_module, "_native_heap_recorder_reset_last_update", _native_heap_recorder_reset_last_update, 1);
+  rb_define_singleton_method(testing_module, "_native_recorder_after_gc_step", _native_recorder_after_gc_step, 1);
 
   ok_symbol = ID2SYM(rb_intern_const("ok"));
   error_symbol = ID2SYM(rb_intern_const("error"));
@@ -330,6 +327,8 @@ static VALUE _native_new(VALUE klass) {
   // Note: Any exceptions raised from this note until the TypedData_Wrap_Struct call will lead to the state memory
   // being leaked.
 
+  state->heap_clean_after_gc_enabled = false;
+
   ddog_prof_Slice_ValueType sample_types = {.ptr = all_value_types, .len = ALL_VALUE_TYPES_COUNT};
 
   initialize_slot_concurrency_control(state);
@@ -411,26 +410,33 @@ static void stack_recorder_typed_data_free(void *state_ptr) {
   ruby_xfree(state);
 }
 
-static VALUE _native_initialize(
-  DDTRACE_UNUSED VALUE _self,
-  VALUE recorder_instance,
-  VALUE cpu_time_enabled,
-  VALUE alloc_samples_enabled,
-  VALUE heap_samples_enabled,
-  VALUE heap_size_enabled,
-  VALUE heap_sample_every,
-  VALUE timeline_enabled
-) {
+static VALUE _native_initialize(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
+  VALUE options;
+  rb_scan_args(argc, argv, "0:", &options);
+  if (options == Qnil) options = rb_hash_new();
+
+  VALUE recorder_instance = rb_hash_fetch(options, ID2SYM(rb_intern("self_instance")));
+  VALUE cpu_time_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("cpu_time_enabled")));
+  VALUE alloc_samples_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("alloc_samples_enabled")));
+  VALUE heap_samples_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("heap_samples_enabled")));
+  VALUE heap_size_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("heap_size_enabled")));
+  VALUE heap_sample_every = rb_hash_fetch(options, ID2SYM(rb_intern("heap_sample_every")));
+  VALUE timeline_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("timeline_enabled")));
+  VALUE heap_clean_after_gc_enabled = rb_hash_fetch(options, ID2SYM(rb_intern("heap_clean_after_gc_enabled")));
+
   ENFORCE_BOOLEAN(cpu_time_enabled);
   ENFORCE_BOOLEAN(alloc_samples_enabled);
   ENFORCE_BOOLEAN(heap_samples_enabled);
   ENFORCE_BOOLEAN(heap_size_enabled);
   ENFORCE_TYPE(heap_sample_every, T_FIXNUM);
   ENFORCE_BOOLEAN(timeline_enabled);
+  ENFORCE_BOOLEAN(heap_clean_after_gc_enabled);
 
   struct stack_recorder_state *state;
   TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
 
+  state->heap_clean_after_gc_enabled = (heap_clean_after_gc_enabled == Qtrue);
+
   heap_recorder_set_sample_rate(state->heap_recorder, NUM2INT(heap_sample_every));
 
   uint8_t requested_values_count = ALL_VALUE_TYPES_COUNT -
@@ -675,6 +681,13 @@ void record_endpoint(VALUE recorder_instance, uint64_t local_root_span_id, ddog_
   }
 }
 
+void recorder_after_gc_step(VALUE recorder_instance) {
+  struct stack_recorder_state *state;
+  TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
+
+  if (state->heap_clean_after_gc_enabled) heap_recorder_update_young_objects(state->heap_recorder);
+}
+
 #define MAX_LEN_HEAP_ITERATION_ERROR_MSG 256
 
 // Heap recorder iteration context allows us access to stack recorder state and profile being serialized
@@ -1057,3 +1070,26 @@ static VALUE build_profile_stats(profile_slot *slot, long serialization_time_ns,
   for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(stats_as_hash, arguments[i], arguments[i+1]);
   return stats_as_hash;
 }
+
+static VALUE _native_is_object_recorded(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance, VALUE obj_id) {
+  ENFORCE_TYPE(obj_id, T_FIXNUM);
+
+  struct stack_recorder_state *state;
+  TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
+
+  return heap_recorder_testonly_is_object_recorded(state->heap_recorder, obj_id);
+}
+
+static VALUE _native_heap_recorder_reset_last_update(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance) {
+  struct stack_recorder_state *state;
+  TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
+
+  heap_recorder_testonly_reset_last_update(state->heap_recorder);
+
+  return Qtrue;
+}
+
+static VALUE _native_recorder_after_gc_step(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance) {
+  recorder_after_gc_step(recorder_instance);
+  return Qtrue;
+}

data/ext/datadog_profiling_native_extension/stack_recorder.h

@@ -27,4 +27,5 @@ typedef struct sample_labels {
 void record_sample(VALUE recorder_instance, ddog_prof_Slice_Location locations, sample_values values, sample_labels labels);
 void record_endpoint(VALUE recorder_instance, uint64_t local_root_span_id, ddog_CharSlice endpoint);
 void track_object(VALUE recorder_instance, VALUE new_object, unsigned int sample_weight, ddog_CharSlice *alloc_class);
+void recorder_after_gc_step(VALUE recorder_instance);
 VALUE enforce_recorder_instance(VALUE object);

data/ext/libdatadog_extconf_helpers.rb

@@ -8,7 +8,7 @@ module Datadog
   module LibdatadogExtconfHelpers
     # Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
     # may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
-    LIBDATADOG_VERSION = '~> 12.0.0.1.0'
+    LIBDATADOG_VERSION = '~> 13.1.0.1.0'
 
     # Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
     # libdatadog are moved after the extension gets compiled.

data/lib/datadog/appsec/configuration/settings.rb

@@ -197,6 +197,14 @@ module Datadog
                 o.type :bool, nilable: true
                 o.env 'DD_APPSEC_SCA_ENABLED'
               end
+
+              settings :standalone do
+                option :enabled do |o|
+                  o.type :bool
+                  o.env 'DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED'
+                  o.default false
+                end
+              end
             end
           end
         end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -38,11 +38,7 @@ module Datadog
                           actions: result.actions
                         }
 
-                        if scope.service_entry_span
-                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
-                          scope.service_entry_span.set_tag('appsec.event', 'true')
-                        end
-
+                        Datadog::AppSec::Event.tag_and_keep!(scope, result)
                         scope.processor_context.events << event
                       end