datadog 2.0.0.beta2 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c98feb8dee5da784f25da37c6989ed32929eeac8189590c0aa6763303ac5076d
-  data.tar.gz: 9324b595c888213affe2a342c4be4dcb49643328cdf3ab0aab9616c41a9b5dd6
+  metadata.gz: 5c8f4fef81d3a238299c70c27f6ece67dc31e76e4b6ad2180a2d1126e4c9545a
+  data.tar.gz: 17bc14574f265b2fdba65d4030e37e513f6af33af6ce8be77e1bbb535d4f6ae4
 SHA512:
-  metadata.gz: f4ee11c65fe76ee70b5c85d67de21a5feadc6938976a0869ae230a6d3a28c38d5ed2c8516633d2046f7fb311ed19d62de9a46317e70a1ade2d4147c139693a44
-  data.tar.gz: 05b8bb6ec82ddb7cd7bce5e461c5e48416df745a23cfcc724d7b2cc0de86e047a4b0e8b7d26dbf9f0b62a12f94dc0492937da3676adb97e458fb41a501a005f3
+  metadata.gz: fff160e4a8e316ec82381e5ae6ef5687a434aa0798debcc689c37e127d688e3d0c0f1009befb6871096c77032caeed348df777ba9baab4c8f7d6f4582d02c8bd
+  data.tar.gz: 6fb4d250c8838a101a489cc10ee0672c23dd4a9793ad850b17bc2c375881c0e3e4cf2c7c3bf5645a6ce8bce1e59c06cbb82609dffe489101fb3ba158d655f034

data/CHANGELOG.md

@@ -2,6 +2,44 @@
 
 ## [Unreleased]
 
+## [2.0.0.rc1] - 2024-05-24
+
+### Added
+
+* Core: Add libdatadog crash tracker ([#3384][])
+* OpenTelemetry: Add support for Span Links ([#3572][])
+* Profiling: Enable endpoint profiling for Sidekiq and other background job processors ([#3610][])
+* Tracing: Add dynamically configurable sampling rules ([#3598][])
+* Tracing: Add sampling rule glob pattern matching ([#3616][])
+
+### Changed
+
+* Appsec: Fix undefined method error when Tracing disabled ([#3645][])
+* Profiling: Upgrade to libdatadog 9 ([#3627][])
+* Tracing: Cache ActiveRecord configuration resolver ([#3630][])
+
+### Fixed
+
+* Core: Fix error during telemetry debug logging attempt ([#3617][])
+* OpenTelemetry: Fix attribute merge with Datadog tags ([#3651][])
+* Tracing: Fix environment logger repeated entries ([#3624][])
+
+### Removed
+
+* Profiling: Remove profiler support for Ruby 2.3 and 2.4 ([#3621][])
+* Profiling: Remove deprecated profiler settings ([#3597][])
+
+## [1.23.0] - 2024-05-09
+
+### Added
+
+* Profiling: Enable endpoint profiling for Sidekiq and similar background job processors ([#3619][])
+
+### Fixed
+
+* Fix no such file or directory issue when using single step instrumentation ([#3623][])
+* Fix error during telemetry debug logging attempt ([#3618][])
+
 ## [2.0.0.beta2] - 2024-04-18
 
 ### Added
@@ -2850,7 +2888,9 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.22.0...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0.rc1...master
+[2.0.0.rc1]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0.beta2...v2.0.0.rc1
+[1.23.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.22.0...v1.23.0
 [2.0.0.beta2]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0.beta1...v2.0.0.beta2
 [1.22.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...v1.22.0
 [2.0.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...v2.0.0.beta1
@@ -4145,6 +4185,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3370]: https://github.com/DataDog/dd-trace-rb/issues/3370
 [#3373]: https://github.com/DataDog/dd-trace-rb/issues/3373
 [#3374]: https://github.com/DataDog/dd-trace-rb/issues/3374
+[#3384]: https://github.com/DataDog/dd-trace-rb/issues/3384
 [#3386]: https://github.com/DataDog/dd-trace-rb/issues/3386
 [#3388]: https://github.com/DataDog/dd-trace-rb/issues/3388
 [#3392]: https://github.com/DataDog/dd-trace-rb/issues/3392
@@ -4202,10 +4243,25 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3551]: https://github.com/DataDog/dd-trace-rb/issues/3551
 [#3558]: https://github.com/DataDog/dd-trace-rb/issues/3558
 [#3565]: https://github.com/DataDog/dd-trace-rb/issues/3565
+[#3572]: https://github.com/DataDog/dd-trace-rb/issues/3572
 [#3573]: https://github.com/DataDog/dd-trace-rb/issues/3573
 [#3582]: https://github.com/DataDog/dd-trace-rb/issues/3582
 [#3585]: https://github.com/DataDog/dd-trace-rb/issues/3585
 [#3587]: https://github.com/DataDog/dd-trace-rb/issues/3587
+[#3597]: https://github.com/DataDog/dd-trace-rb/issues/3597
+[#3598]: https://github.com/DataDog/dd-trace-rb/issues/3598
+[#3610]: https://github.com/DataDog/dd-trace-rb/issues/3610
+[#3616]: https://github.com/DataDog/dd-trace-rb/issues/3616
+[#3617]: https://github.com/DataDog/dd-trace-rb/issues/3617
+[#3618]: https://github.com/DataDog/dd-trace-rb/issues/3618
+[#3619]: https://github.com/DataDog/dd-trace-rb/issues/3619
+[#3621]: https://github.com/DataDog/dd-trace-rb/issues/3621
+[#3623]: https://github.com/DataDog/dd-trace-rb/issues/3623
+[#3624]: https://github.com/DataDog/dd-trace-rb/issues/3624
+[#3627]: https://github.com/DataDog/dd-trace-rb/issues/3627
+[#3630]: https://github.com/DataDog/dd-trace-rb/issues/3630
+[#3645]: https://github.com/DataDog/dd-trace-rb/issues/3645
+[#3651]: https://github.com/DataDog/dd-trace-rb/issues/3651
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot

data/ext/datadog_profiling_native_extension/NativeExtensionDesign.md

@@ -28,7 +28,7 @@ documentation.**
 The profiling native extension is (and must always be) designed to **not cause failures** during gem installation, even
 if some features, Ruby versions, or operating systems are not supported.
 
-E.g. the extension must not break installation on Ruby 2.1 (or the oldest Ruby version we support at the time) on 64-bit ARM macOS,
+E.g. the extension must not break installation on Ruby 2.5 (or the oldest Ruby version we support at the time) on 64-bit ARM macOS,
 even if at run time it will effectively do nothing for such a setup.
 
 We have a CI setup to help validate this, but this is really important to keep in mind when adding to or changing the

data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c

@@ -929,18 +929,6 @@ static VALUE _native_stats(DDTRACE_UNUSED VALUE self, VALUE instance) {
   struct cpu_and_wall_time_worker_state *state;
   TypedData_Get_Struct(instance, struct cpu_and_wall_time_worker_state, &cpu_and_wall_time_worker_typed_data, state);
 
-  VALUE pretty_cpu_sampling_time_ns_min = state->stats.cpu_sampling_time_ns_min == UINT64_MAX ? Qnil : ULL2NUM(state->stats.cpu_sampling_time_ns_min);
-  VALUE pretty_cpu_sampling_time_ns_max = state->stats.cpu_sampling_time_ns_max == 0 ? Qnil : ULL2NUM(state->stats.cpu_sampling_time_ns_max);
-  VALUE pretty_cpu_sampling_time_ns_total = state->stats.cpu_sampling_time_ns_total == 0 ? Qnil : ULL2NUM(state->stats.cpu_sampling_time_ns_total);
-  VALUE pretty_cpu_sampling_time_ns_avg =
-    state->stats.cpu_sampled == 0 ? Qnil : DBL2NUM(((double) state->stats.cpu_sampling_time_ns_total) / state->stats.cpu_sampled);
-
-  VALUE pretty_allocation_sampling_time_ns_min = state->stats.allocation_sampling_time_ns_min == UINT64_MAX ? Qnil : ULL2NUM(state->stats.allocation_sampling_time_ns_min);
-  VALUE pretty_allocation_sampling_time_ns_max = state->stats.allocation_sampling_time_ns_max == 0 ? Qnil : ULL2NUM(state->stats.allocation_sampling_time_ns_max);
-  VALUE pretty_allocation_sampling_time_ns_total = state->stats.allocation_sampling_time_ns_total == 0 ? Qnil : ULL2NUM(state->stats.allocation_sampling_time_ns_total);
-  VALUE pretty_allocation_sampling_time_ns_avg =
-    state->stats.allocation_sampled == 0 ? Qnil : DBL2NUM(((double) state->stats.allocation_sampling_time_ns_total) / state->stats.allocation_sampled);
-
   unsigned long total_cpu_samples_attempted = state->stats.cpu_sampled + state->stats.cpu_skipped;
   VALUE effective_cpu_sample_rate =
     total_cpu_samples_attempted == 0 ? Qnil : DBL2NUM(((double) state->stats.cpu_sampled) / total_cpu_samples_attempted);
@@ -968,19 +956,19 @@ static VALUE _native_stats(DDTRACE_UNUSED VALUE self, VALUE instance) {
     ID2SYM(rb_intern("cpu_sampled")),                /* => */ UINT2NUM(state->stats.cpu_sampled),
     ID2SYM(rb_intern("cpu_skipped")),                /* => */ UINT2NUM(state->stats.cpu_skipped),
     ID2SYM(rb_intern("cpu_effective_sample_rate")),  /* => */ effective_cpu_sample_rate,
-    ID2SYM(rb_intern("cpu_sampling_time_ns_min")),   /* => */ pretty_cpu_sampling_time_ns_min,
-    ID2SYM(rb_intern("cpu_sampling_time_ns_max")),   /* => */ pretty_cpu_sampling_time_ns_max,
-    ID2SYM(rb_intern("cpu_sampling_time_ns_total")), /* => */ pretty_cpu_sampling_time_ns_total,
-    ID2SYM(rb_intern("cpu_sampling_time_ns_avg")),   /* => */ pretty_cpu_sampling_time_ns_avg,
+    ID2SYM(rb_intern("cpu_sampling_time_ns_min")),   /* => */ RUBY_NUM_OR_NIL(state->stats.cpu_sampling_time_ns_min, != UINT64_MAX, ULL2NUM),
+    ID2SYM(rb_intern("cpu_sampling_time_ns_max")),   /* => */ RUBY_NUM_OR_NIL(state->stats.cpu_sampling_time_ns_max, > 0, ULL2NUM),
+    ID2SYM(rb_intern("cpu_sampling_time_ns_total")), /* => */ RUBY_NUM_OR_NIL(state->stats.cpu_sampling_time_ns_total, > 0, ULL2NUM),
+    ID2SYM(rb_intern("cpu_sampling_time_ns_avg")),   /* => */ RUBY_AVG_OR_NIL(state->stats.cpu_sampling_time_ns_total, state->stats.cpu_sampled),
 
     // Allocation stats
     ID2SYM(rb_intern("allocation_sampled")),                /* => */ state->allocation_profiling_enabled ? ULONG2NUM(state->stats.allocation_sampled) : Qnil,
     ID2SYM(rb_intern("allocation_skipped")),                /* => */ state->allocation_profiling_enabled ? ULONG2NUM(state->stats.allocation_skipped) : Qnil,
     ID2SYM(rb_intern("allocation_effective_sample_rate")),  /* => */ effective_allocation_sample_rate,
-    ID2SYM(rb_intern("allocation_sampling_time_ns_min")),   /* => */ pretty_allocation_sampling_time_ns_min,
-    ID2SYM(rb_intern("allocation_sampling_time_ns_max")),   /* => */ pretty_allocation_sampling_time_ns_max,
-    ID2SYM(rb_intern("allocation_sampling_time_ns_total")), /* => */ pretty_allocation_sampling_time_ns_total,
-    ID2SYM(rb_intern("allocation_sampling_time_ns_avg")),   /* => */ pretty_allocation_sampling_time_ns_avg,
+    ID2SYM(rb_intern("allocation_sampling_time_ns_min")),   /* => */ RUBY_NUM_OR_NIL(state->stats.allocation_sampling_time_ns_min, != UINT64_MAX, ULL2NUM),
+    ID2SYM(rb_intern("allocation_sampling_time_ns_max")),   /* => */ RUBY_NUM_OR_NIL(state->stats.allocation_sampling_time_ns_max, > 0, ULL2NUM),
+    ID2SYM(rb_intern("allocation_sampling_time_ns_total")), /* => */ RUBY_NUM_OR_NIL(state->stats.allocation_sampling_time_ns_total, > 0, ULL2NUM),
+    ID2SYM(rb_intern("allocation_sampling_time_ns_avg")),   /* => */ RUBY_AVG_OR_NIL(state->stats.allocation_sampling_time_ns_total, state->stats.allocation_sampled),
     ID2SYM(rb_intern("allocation_sampler_snapshot")),       /* => */ allocation_sampler_snapshot,
     ID2SYM(rb_intern("allocations_during_sample")),         /* => */ state->allocation_profiling_enabled ? UINT2NUM(state->stats.allocations_during_sample) : Qnil,
   };

data/ext/datadog_profiling_native_extension/collectors_thread_context.c

@@ -217,7 +217,7 @@ static long thread_id_for(VALUE thread);
 static VALUE _native_stats(VALUE self, VALUE collector_instance);
 static VALUE _native_gc_tracking(VALUE self, VALUE collector_instance);
 static void trace_identifiers_for(struct thread_context_collector_state *state, VALUE thread, struct trace_identifiers *trace_identifiers_result);
-static bool should_collect_resource(VALUE root_span_type);
+static bool should_collect_resource(VALUE root_span);
 static VALUE _native_reset_after_fork(DDTRACE_UNUSED VALUE self, VALUE collector_instance);
 static VALUE thread_list(struct thread_context_collector_state *state);
 static VALUE _native_sample_allocation(DDTRACE_UNUSED VALUE self, VALUE collector_instance, VALUE sample_weight, VALUE new_object);
@@ -1146,10 +1146,7 @@ static void trace_identifiers_for(struct thread_context_collector_state *state,
 
   trace_identifiers_result->valid = true;
 
-  if (!state->endpoint_collection_enabled) return;
-
-  VALUE root_span_type = rb_ivar_get(root_span, at_type_id /* @type */);
-  if (root_span_type == Qnil || !should_collect_resource(root_span_type)) return;
+  if (!state->endpoint_collection_enabled || !should_collect_resource(root_span)) return;
 
   VALUE trace_resource = rb_ivar_get(active_trace, at_resource_id /* @resource */);
   if (RB_TYPE_P(trace_resource, T_STRING)) {
@@ -1160,21 +1157,32 @@ static void trace_identifiers_for(struct thread_context_collector_state *state,
   }
 }
 
-// We only collect the resource for spans of types:
+// We opt-in to collecting the resource for spans of types:
 // * 'web', for web requests
-// * proxy', used by the rack integration with request_queuing: true (e.g. also represents a web request)
+// * 'proxy', used by the rack integration with request_queuing: true (e.g. also represents a web request)
+// * 'worker', used for sidekiq and similar background job processors
 //
-// NOTE: Currently we're only interested in HTTP service endpoints. Over time, this list may be expanded.
+// Over time, this list may be expanded.
 // Resources MUST NOT include personal identifiable information (PII); this should not be the case with
 // ddtrace integrations, but worth mentioning just in case :)
-static bool should_collect_resource(VALUE root_span_type) {
+static bool should_collect_resource(VALUE root_span) {
+  VALUE root_span_type = rb_ivar_get(root_span, at_type_id /* @type */);
+  if (root_span_type == Qnil) return false;
   ENFORCE_TYPE(root_span_type, T_STRING);
 
   int root_span_type_length = RSTRING_LEN(root_span_type);
   const char *root_span_type_value = StringValuePtr(root_span_type);
 
-  return (root_span_type_length == strlen("web") && (memcmp("web", root_span_type_value, strlen("web")) == 0)) ||
+  bool is_web_request =
+    (root_span_type_length == strlen("web") && (memcmp("web", root_span_type_value, strlen("web")) == 0)) ||
     (root_span_type_length == strlen("proxy") && (memcmp("proxy", root_span_type_value, strlen("proxy")) == 0));
+
+  if (is_web_request) return true;
+
+  bool is_worker_request =
+    (root_span_type_length == strlen("worker") && (memcmp("worker", root_span_type_value, strlen("worker")) == 0));
+
+  return is_worker_request;
 }
 
 // After the Ruby VM forks, this method gets called in the child process to clean up any leftover state from the parent.

data/ext/datadog_profiling_native_extension/crashtracker.c

@@ -0,0 +1,108 @@
+#include <ruby.h>
+#include <datadog/common.h>
+#include <libdatadog_helpers.h>
+
+static VALUE _native_start_or_update_on_fork(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self);
+static VALUE _native_stop(DDTRACE_UNUSED VALUE _self);
+
+// Used to report Ruby VM crashes.
+// Once initialized, segfaults will be reported automatically using libdatadog.
+
+void crashtracker_init(VALUE profiling_module) {
+  VALUE crashtracker_class = rb_define_class_under(profiling_module, "Crashtracker", rb_cObject);
+
+  rb_define_singleton_method(crashtracker_class, "_native_start_or_update_on_fork", _native_start_or_update_on_fork, -1);
+  rb_define_singleton_method(crashtracker_class, "_native_stop", _native_stop, 0);
+}
+
+static VALUE _native_start_or_update_on_fork(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
+  VALUE options;
+  rb_scan_args(argc, argv, "0:", &options);
+
+  VALUE exporter_configuration = rb_hash_fetch(options, ID2SYM(rb_intern("exporter_configuration")));
+  VALUE path_to_crashtracking_receiver_binary = rb_hash_fetch(options, ID2SYM(rb_intern("path_to_crashtracking_receiver_binary")));
+  VALUE ld_library_path = rb_hash_fetch(options, ID2SYM(rb_intern("ld_library_path")));
+  VALUE tags_as_array = rb_hash_fetch(options, ID2SYM(rb_intern("tags_as_array")));
+  VALUE action = rb_hash_fetch(options, ID2SYM(rb_intern("action")));
+  VALUE upload_timeout_seconds = rb_hash_fetch(options, ID2SYM(rb_intern("upload_timeout_seconds")));
+
+  VALUE start_action = ID2SYM(rb_intern("start"));
+  VALUE update_on_fork_action = ID2SYM(rb_intern("update_on_fork"));
+
+  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
+  ENFORCE_TYPE(tags_as_array, T_ARRAY);
+  ENFORCE_TYPE(path_to_crashtracking_receiver_binary, T_STRING);
+  ENFORCE_TYPE(ld_library_path, T_STRING);
+  ENFORCE_TYPE(action, T_SYMBOL);
+  ENFORCE_TYPE(upload_timeout_seconds, T_FIXNUM);
+
+  if (action != start_action && action != update_on_fork_action) rb_raise(rb_eArgError, "Unexpected action: %+"PRIsVALUE, action);
+
+  VALUE version = ddtrace_version();
+  ddog_prof_Endpoint endpoint = endpoint_from(exporter_configuration);
+
+  // Tags are heap-allocated, so after here we can't raise exceptions otherwise we'll leak this memory
+  // Start of exception-free zone to prevent leaks {{
+  ddog_Vec_Tag tags = convert_tags(tags_as_array);
+
+  ddog_prof_CrashtrackerConfiguration config = {
+    .additional_files = {},
+    // The Ruby VM already uses an alt stack to detect stack overflows so the crash handler must not overwrite it.
+    //
+    // @ivoanjo: Specifically, with `create_alt_stack = true` I saw a segfault, such as Ruby 2.6's bug with
+    // "Process.detach(fork { exit! }).instance_variable_get(:@foo)" being turned into a
+    // "-e:1:in `instance_variable_get': stack level too deep (SystemStackError)" by Ruby.
+    //
+    // The Ruby crash handler also seems to get confused when this option is enabled and
+    // "Process.kill('SEGV', Process.pid)" gets run.
+    .create_alt_stack = false,
+    .endpoint = endpoint,
+    .resolve_frames = DDOG_PROF_STACKTRACE_COLLECTION_ENABLED,
+    .timeout_secs = FIX2INT(upload_timeout_seconds),
+  };
+
+  ddog_prof_CrashtrackerMetadata metadata = {
+    .profiling_library_name = DDOG_CHARSLICE_C("dd-trace-rb"),
+    .profiling_library_version = char_slice_from_ruby_string(version),
+    .family = DDOG_CHARSLICE_C("ruby"),
+    .tags = &tags,
+  };
+
+  ddog_prof_EnvVar ld_library_path_env = {
+    .key = DDOG_CHARSLICE_C("LD_LIBRARY_PATH"),
+    .val = char_slice_from_ruby_string(ld_library_path),
+  };
+
+  ddog_prof_CrashtrackerReceiverConfig receiver_config = {
+    .args = {},
+    .env = {.ptr = &ld_library_path_env, .len = 1},
+    .path_to_receiver_binary = char_slice_from_ruby_string(path_to_crashtracking_receiver_binary),
+    .optional_stderr_filename = {},
+    .optional_stdout_filename = {},
+  };
+
+  ddog_prof_CrashtrackerResult result =
+    action == start_action ?
+      ddog_prof_Crashtracker_init(config, receiver_config, metadata) :
+      ddog_prof_Crashtracker_update_on_fork(config, receiver_config, metadata);
+
+  // Clean up before potentially raising any exceptions
+  ddog_Vec_Tag_drop(tags);
+  // }} End of exception-free zone to prevent leaks
+
+  if (result.tag == DDOG_PROF_CRASHTRACKER_RESULT_ERR) {
+    rb_raise(rb_eRuntimeError, "Failed to start/update the crash tracker: %"PRIsVALUE, get_error_details_and_drop(&result.err));
+  }
+
+  return Qtrue;
+}
+
+static VALUE _native_stop(DDTRACE_UNUSED VALUE _self) {
+  ddog_prof_CrashtrackerResult result = ddog_prof_Crashtracker_shutdown();
+
+  if (result.tag == DDOG_PROF_CRASHTRACKER_RESULT_ERR) {
+    rb_raise(rb_eRuntimeError, "Failed to stop the crash tracker: %"PRIsVALUE, get_error_details_and_drop(&result.err));
+  }
+
+  return Qtrue;
+}

data/ext/datadog_profiling_native_extension/extconf.rb

@@ -126,7 +126,7 @@ if RUBY_PLATFORM.include?('linux')
   # have_library 'pthread'
   # have_func 'pthread_getcpuclockid'
   # ```
-  # but a) it broke the build on Windows, b) on older Ruby versions (2.2 and below) and c) It's slower to build
+  # but it's slower to build
   # so instead we just assume that we have the function we need on Linux, and nowhere else
   $defs << '-DHAVE_PTHREAD_GETCPUCLOCKID'
 end
@@ -163,6 +163,11 @@ $defs << '-DNO_THREAD_TID' if RUBY_VERSION < '3.1'
 # On older Rubies, there was no jit_return member on the rb_control_frame_t struct
 $defs << '-DNO_JIT_RETURN' if RUBY_VERSION < '3.1'
 
+# On older Rubies, rb_gc_force_recycle allowed to free objects in a way that
+# would be invisible to free tracepoints, finalizers and without cleaning
+# obj_to_id_tbl mappings.
+$defs << '-DHAVE_WORKING_RB_GC_FORCE_RECYCLE' if RUBY_VERSION < '3.1'
+
 # On older Rubies, we need to use a backported version of this function. See private_vm_api_access.h for details.
 $defs << '-DUSE_BACKPORTED_RB_PROFILE_FRAME_METHOD_NAME' if RUBY_VERSION < '3'
 
@@ -175,34 +180,15 @@ $defs << '-DNO_IMEMO_NAME' if RUBY_VERSION < '3'
 # On older Rubies, objects would not move
 $defs << '-DNO_T_MOVED' if RUBY_VERSION < '2.7'
 
+# On older Rubies, there was no RUBY_SEEN_OBJ_ID flag
+$defs << '-DNO_SEEN_OBJ_ID_FLAG' if RUBY_VERSION < '2.7'
+
 # On older Rubies, rb_global_vm_lock_struct did not include the owner field
 $defs << '-DNO_GVL_OWNER' if RUBY_VERSION < '2.6'
 
 # On older Rubies, there was no thread->invoke_arg
 $defs << '-DNO_THREAD_INVOKE_ARG' if RUBY_VERSION < '2.6'
 
-# On older Rubies, we need to use rb_thread_t instead of rb_execution_context_t
-$defs << '-DUSE_THREAD_INSTEAD_OF_EXECUTION_CONTEXT' if RUBY_VERSION < '2.5'
-
-# On older Rubies, extensions can't use GET_VM()
-$defs << '-DNO_GET_VM' if RUBY_VERSION < '2.5'
-
-# On older Rubies...
-if RUBY_VERSION < '2.4'
-  # ...we need to use RUBY_VM_NORMAL_ISEQ_P instead of VM_FRAME_RUBYFRAME_P
-  $defs << '-DUSE_ISEQ_P_INSTEAD_OF_RUBYFRAME_P'
-  # ...we use a legacy copy of rb_vm_frame_method_entry
-  $defs << '-DUSE_LEGACY_RB_VM_FRAME_METHOD_ENTRY'
-end
-
-# On older Rubies, rb_gc_force_recycle allowed to free objects in a way that
-# would be invisible to free tracepoints, finalizers and without cleaning
-# obj_to_id_tbl mappings.
-$defs << '-DHAVE_WORKING_RB_GC_FORCE_RECYCLE' if RUBY_VERSION < '3.1'
-
-# On older Rubies, there was no RUBY_SEEN_OBJ_ID flag
-$defs << '-DNO_SEEN_OBJ_ID_FLAG' if RUBY_VERSION < '2.7'
-
 # If we got here, libdatadog is available and loaded
 ENV['PKG_CONFIG_PATH'] = "#{ENV['PKG_CONFIG_PATH']}:#{Libdatadog.pkgconfig_folder}"
 Logging.message("[datadog] PKG_CONFIG_PATH set to #{ENV['PKG_CONFIG_PATH'].inspect}\n")

data/ext/datadog_profiling_native_extension/heap_recorder.c

@@ -158,6 +158,13 @@ struct heap_recorder {
 
   // Sampling state
   uint num_recordings_skipped;
+
+  struct stats_last_update {
+    size_t objects_alive;
+    size_t objects_dead;
+    size_t objects_skipped;
+    size_t objects_frozen;
+  } stats_last_update;
 };
 static heap_record* get_or_create_heap_record(heap_recorder*, ddog_prof_Slice_Location);
 static void cleanup_heap_record_if_unused(heap_recorder*, heap_record*);
@@ -372,6 +379,9 @@ void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
     rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
   }
 
+  // Reset last update stats, we'll be building them from scratch during the st_foreach call below
+  heap_recorder->stats_last_update = (struct stats_last_update) {};
+
   st_foreach(heap_recorder->object_records, st_object_record_update, (st_data_t) heap_recorder);
 
   heap_recorder->object_records_snapshot = st_copy(heap_recorder->object_records);
@@ -427,6 +437,22 @@ bool heap_recorder_for_each_live_object(
   return true;
 }
 
+VALUE heap_recorder_state_snapshot(heap_recorder *heap_recorder) {
+  VALUE arguments[] = {
+    ID2SYM(rb_intern("num_object_records")), /* => */ LONG2NUM(heap_recorder->object_records->num_entries),
+    ID2SYM(rb_intern("num_heap_records")),   /* => */ LONG2NUM(heap_recorder->heap_records->num_entries),
+
+    // Stats as of last update
+    ID2SYM(rb_intern("last_update_objects_alive")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_alive),
+    ID2SYM(rb_intern("last_update_objects_dead")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_dead),
+    ID2SYM(rb_intern("last_update_objects_skipped")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_skipped),
+    ID2SYM(rb_intern("last_update_objects_frozen")), /* => */ LONG2NUM(heap_recorder->stats_last_update.objects_frozen),
+  };
+  VALUE hash = rb_hash_new();
+  for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(hash, arguments[i], arguments[i+1]);
+  return hash;
+}
+
 void heap_recorder_testonly_assert_hash_matches(ddog_prof_Slice_Location locations) {
   heap_stack *stack = heap_stack_new(locations);
   heap_record_key stack_based_key = (heap_record_key) {
@@ -497,12 +523,14 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
     // no point checking for liveness or updating its size, so exit early.
     // NOTE: This means that there should be an equivalent check during actual
     //       iteration otherwise we'd iterate/expose stale object data.
+    recorder->stats_last_update.objects_skipped++;
     return ST_CONTINUE;
   }
 
   if (!ruby_ref_from_id(LONG2NUM(obj_id), &ref)) {
     // Id no longer associated with a valid ref. Need to delete this object record!
     on_committed_object_record_cleanup(recorder, record);
+    recorder->stats_last_update.objects_dead++;
     return ST_DELETE;
   }
 
@@ -537,6 +565,7 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
       RB_FL_SET(ref, RUBY_FL_SEEN_OBJ_ID);
 
       on_committed_object_record_cleanup(recorder, record);
+      recorder->stats_last_update.objects_dead++;
       return ST_DELETE;
     }
 
@@ -550,6 +579,11 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
     record->object_data.is_frozen = RB_OBJ_FROZEN(ref);
   }
 
+  recorder->stats_last_update.objects_alive++;
+  if (record->object_data.is_frozen) {
+    recorder->stats_last_update.objects_frozen++;
+  }
+
   return ST_CONTINUE;
 }
 
@@ -767,9 +801,10 @@ void object_record_free(object_record *record) {
 
 VALUE object_record_inspect(object_record *record) {
   heap_frame top_frame = record->heap_record->stack->frames[0];
-  VALUE inspect = rb_sprintf("obj_id=%ld weight=%d size=%zu location=%s:%d alloc_gen=%zu gen_age=%zu ",
-      record->obj_id, record->object_data.weight, record->object_data.size, top_frame.filename,
-      (int) top_frame.line, record->object_data.alloc_gen, record->object_data.gen_age);
+  live_object_data object_data = record->object_data;
+  VALUE inspect = rb_sprintf("obj_id=%ld weight=%d size=%zu location=%s:%d alloc_gen=%zu gen_age=%zu frozen=%d ",
+      record->obj_id, object_data.weight, object_data.size, top_frame.filename,
+      (int) top_frame.line, object_data.alloc_gen, object_data.gen_age, object_data.is_frozen);
 
   const char *class = record->object_data.class;
   if (class != NULL) {

data/ext/datadog_profiling_native_extension/heap_recorder.h

@@ -150,6 +150,11 @@ bool heap_recorder_for_each_live_object(
     bool (*for_each_callback)(heap_recorder_iteration_data data, void* extra_arg),
     void *for_each_callback_extra_arg);
 
+// Return a Ruby hash containing a snapshot of this recorder's interesting state at calling time.
+// WARN: This allocates in the Ruby VM and therefore should not be called without the
+//       VM lock or during GC.
+VALUE heap_recorder_state_snapshot(heap_recorder *heap_recorder);
+
 // v--- TEST-ONLY APIs ---v
 
 // Assert internal hashing logic is valid for the provided locations and its

data/ext/datadog_profiling_native_extension/http_transport.c

@@ -11,11 +11,6 @@
 static VALUE ok_symbol = Qnil; // :ok in Ruby
 static VALUE error_symbol = Qnil; // :error in Ruby
 
-static ID agentless_id; // id of :agentless in Ruby
-static ID agent_id; // id of :agent in Ruby
-
-static ID log_failure_to_process_tag_id; // id of :log_failure_to_process_tag in Ruby
-
 static VALUE library_version_string = Qnil;
 
 struct call_exporter_without_gvl_arguments {
@@ -30,9 +25,6 @@ inline static ddog_ByteSlice byte_slice_from_ruby_string(VALUE string);
 static VALUE _native_validate_exporter(VALUE self, VALUE exporter_configuration);
 static ddog_prof_Exporter_NewResult create_exporter(VALUE exporter_configuration, VALUE tags_as_array);
 static VALUE handle_exporter_failure(ddog_prof_Exporter_NewResult exporter_result);
-static ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration);
-static ddog_Vec_Tag convert_tags(VALUE tags_as_array);
-static void safely_log_failure_to_process_tag(ddog_Vec_Tag tags, VALUE err_details);
 static VALUE _native_do_export(
   VALUE self,
   VALUE exporter_configuration,
@@ -60,9 +52,6 @@ void http_transport_init(VALUE profiling_module) {
 
   ok_symbol = ID2SYM(rb_intern_const("ok"));
   error_symbol = ID2SYM(rb_intern_const("error"));
-  agentless_id = rb_intern_const("agentless");
-  agent_id = rb_intern_const("agent");
-  log_failure_to_process_tag_id = rb_intern_const("log_failure_to_process_tag");
 
   library_version_string = ddtrace_version();
   rb_global_variable(&library_version_string);
@@ -116,88 +105,6 @@ static VALUE handle_exporter_failure(ddog_prof_Exporter_NewResult exporter_resul
     rb_ary_new_from_args(2, error_symbol, get_error_details_and_drop(&exporter_result.err));
 }
 
-static ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
-  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
-
-  ID working_mode = SYM2ID(rb_ary_entry(exporter_configuration, 0)); // SYM2ID verifies its input so we can do this safely
-
-  if (working_mode != agentless_id && working_mode != agent_id) {
-    rb_raise(rb_eArgError, "Failed to initialize transport: Unexpected working mode, expected :agentless or :agent");
-  }
-
-  if (working_mode == agentless_id) {
-    VALUE site = rb_ary_entry(exporter_configuration, 1);
-    VALUE api_key = rb_ary_entry(exporter_configuration, 2);
-    ENFORCE_TYPE(site, T_STRING);
-    ENFORCE_TYPE(api_key, T_STRING);
-
-    return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
-  } else { // agent_id
-    VALUE base_url = rb_ary_entry(exporter_configuration, 1);
-    ENFORCE_TYPE(base_url, T_STRING);
-
-    return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
-  }
-}
-
-__attribute__((warn_unused_result))
-static ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
-  ENFORCE_TYPE(tags_as_array, T_ARRAY);
-
-  long tags_count = RARRAY_LEN(tags_as_array);
-  ddog_Vec_Tag tags = ddog_Vec_Tag_new();
-
-  for (long i = 0; i < tags_count; i++) {
-    VALUE name_value_pair = rb_ary_entry(tags_as_array, i);
-
-    if (!RB_TYPE_P(name_value_pair, T_ARRAY)) {
-      ddog_Vec_Tag_drop(tags);
-      ENFORCE_TYPE(name_value_pair, T_ARRAY);
-    }
-
-    // Note: We can index the array without checking its size first because rb_ary_entry returns Qnil if out of bounds
-    VALUE tag_name = rb_ary_entry(name_value_pair, 0);
-    VALUE tag_value = rb_ary_entry(name_value_pair, 1);
-
-    if (!(RB_TYPE_P(tag_name, T_STRING) && RB_TYPE_P(tag_value, T_STRING))) {
-      ddog_Vec_Tag_drop(tags);
-      ENFORCE_TYPE(tag_name, T_STRING);
-      ENFORCE_TYPE(tag_value, T_STRING);
-    }
-
-    ddog_Vec_Tag_PushResult push_result =
-      ddog_Vec_Tag_push(&tags, char_slice_from_ruby_string(tag_name), char_slice_from_ruby_string(tag_value));
-
-    if (push_result.tag == DDOG_VEC_TAG_PUSH_RESULT_ERR) {
-      // libdatadog validates tags and may catch invalid tags that ddtrace didn't actually catch.
-      // We warn users about such tags, and then just ignore them.
-      safely_log_failure_to_process_tag(tags, get_error_details_and_drop(&push_result.err));
-    }
-  }
-
-  return tags;
-}
-
-static VALUE log_failure_to_process_tag(VALUE err_details) {
-  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
-  VALUE profiling_module = rb_const_get(datadog_module, rb_intern("Profiling"));
-  VALUE http_transport_class = rb_const_get(profiling_module, rb_intern("HttpTransport"));
-
-  return rb_funcall(http_transport_class, log_failure_to_process_tag_id, 1, err_details);
-}
-
-// Since we are calling into Ruby code, it may raise an exception. This method ensure that dynamically-allocated tags
-// get cleaned before propagating the exception.
-static void safely_log_failure_to_process_tag(ddog_Vec_Tag tags, VALUE err_details) {
-  int exception_state;
-  rb_protect(log_failure_to_process_tag, err_details, &exception_state);
-
-  if (exception_state) {           // An exception was raised
-    ddog_Vec_Tag_drop(tags); // clean up
-    rb_jump_tag(exception_state);  // "Re-raise" exception
-  }
-}
-
 // Note: This function handles a bunch of libdatadog dynamically-allocated objects, so it MUST not use any Ruby APIs
 // which can raise exceptions, otherwise the objects will be leaked.
 static VALUE perform_export(