ddtrace 1.4.2 → 1.19.0

data/ext/ddtrace_profiling_native_extension/collectors_thread_context.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <ruby.h>
+#include <stdbool.h>
+
+void thread_context_collector_sample(
+  VALUE self_instance,
+  long current_monotonic_wall_time_ns,
+  VALUE profiler_overhead_stack_thread
+);
+void thread_context_collector_sample_allocation(VALUE self_instance, unsigned int sample_weight, VALUE new_object);
+VALUE thread_context_collector_sample_after_gc(VALUE self_instance);
+void thread_context_collector_on_gc_start(VALUE self_instance);
+bool thread_context_collector_on_gc_finish(VALUE self_instance);
+VALUE enforce_thread_context_collector_instance(VALUE object);

data/ext/ddtrace_profiling_native_extension/extconf.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 # rubocop:disable Style/StderrPuts
 # rubocop:disable Style/GlobalVars
 
@@ -28,7 +26,7 @@ def skip_building_extension!(reason)
   if fail_install_if_missing_extension
     require 'mkmf'
     Logging.message(
-      ' [ddtrace] Failure cause: ' \
+      '[ddtrace] Failure cause: ' \
       "#{Datadog::Profiling::NativeExtensionHelpers::Supported.render_skipped_reason_file(**reason)}\n"
     )
   else
@@ -67,6 +65,10 @@ $stderr.puts(
 # that may fail on an environment not properly setup for building Ruby extensions.
 require 'mkmf'
 
+Logging.message("[ddtrace] Using compiler:\n")
+xsystem("#{CONFIG['CC']} -v")
+Logging.message("[ddtrace] End of compiler information\n")
+
 # mkmf on modern Rubies actually has an append_cflags that does something similar
 # (see https://github.com/ruby/ruby/pull/5760), but as usual we need a bit more boilerplate to deal with legacy Rubies
 def add_compiler_flag(flag)
@@ -77,13 +79,17 @@ def add_compiler_flag(flag)
   end
 end
 
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true'
+
 # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
 # * by upstream Ruby -- search for gnu99 in the codebase
 # * by msgpack, another ddtrace dependency
 #   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
 add_compiler_flag '-std=gnu99'
 
-# Gets really noisy when we include the MJIT header, let's omit it
+# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 add_compiler_flag '-Wno-unused-function'
 
 # Allow defining variables at any point in a function
@@ -93,9 +99,6 @@ add_compiler_flag '-Wno-declaration-after-statement'
 # cause a segfault later. Let's ensure that never happens.
 add_compiler_flag '-Werror-implicit-function-declaration'
 
-# Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
-add_compiler_flag '-Wunused-parameter'
-
 # The native extension is not intended to expose any symbols/functions for other native libraries to use;
 # the sole exception being `Init_ddtrace_profiling_native_extension` which needs to be visible for Ruby to call it when
 # it `dlopen`s the library.
@@ -104,30 +107,83 @@ add_compiler_flag '-Wunused-parameter'
 # For more details see https://gcc.gnu.org/wiki/Visibility
 add_compiler_flag '-fvisibility=hidden'
 
+# Avoid legacy C definitions
+add_compiler_flag '-Wold-style-definition'
+
 # Enable all other compiler warnings
 add_compiler_flag '-Wall'
 add_compiler_flag '-Wextra'
 
+if ENV['DDTRACE_DEBUG']
+  CONFIG['optflags'] = '-O0'
+  CONFIG['debugflags'] = '-ggdb3'
+end
+
 if RUBY_PLATFORM.include?('linux')
   # Supposedly, the correct way to do this is
   # ```
   # have_library 'pthread'
   # have_func 'pthread_getcpuclockid'
   # ```
-  # but it broke the build on Windows and on older Ruby versions (2.2)
+  # but a) it broke the build on Windows, b) on older Ruby versions (2.2 and below) and c) It's slower to build
   # so instead we just assume that we have the function we need on Linux, and nowhere else
   $defs << '-DHAVE_PTHREAD_GETCPUCLOCKID'
 end
 
+# On older Rubies, rb_postponed_job_preregister/rb_postponed_job_trigger did not exist
+$defs << '-DNO_POSTPONED_TRIGGER' if RUBY_VERSION < '3.3'
+
+# On older Rubies, M:N threads were not available
+$defs << '-DNO_MN_THREADS_AVAILABLE' if RUBY_VERSION < '3.3'
+
+# On older Rubies, we did not need to include the ractor header (this was built into the MJIT header)
+$defs << '-DNO_RACTOR_HEADER_INCLUDE' if RUBY_VERSION < '3.3'
+
+# On older Rubies, some of the Ractor internal APIs were directly accessible
+$defs << '-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY' if RUBY_VERSION < '3.3'
+
 # On older Rubies, there was no struct rb_native_thread. See private_vm_api_acccess.c for details.
 $defs << '-DNO_RB_NATIVE_THREAD' if RUBY_VERSION < '3.2'
 
+# On older Rubies, there was no struct rb_thread_sched (it was struct rb_global_vm_lock_struct)
+$defs << '-DNO_RB_THREAD_SCHED' if RUBY_VERSION < '3.2'
+
+# On older Rubies, the first_lineno inside a location was a VALUE and not a int (https://github.com/ruby/ruby/pull/6430)
+$defs << '-DNO_INT_FIRST_LINENO' if RUBY_VERSION < '3.2'
+
+# On older Rubies, "pop" was not a primitive operation
+$defs << '-DNO_PRIMITIVE_POP' if RUBY_VERSION < '3.2'
+
+# On older Rubies, there was no tid member in the internal thread structure
+$defs << '-DNO_THREAD_TID' if RUBY_VERSION < '3.1'
+
+# On older Rubies, there was no jit_return member on the rb_control_frame_t struct
+$defs << '-DNO_JIT_RETURN' if RUBY_VERSION < '3.1'
+
 # On older Rubies, we need to use a backported version of this function. See private_vm_api_access.h for details.
 $defs << '-DUSE_BACKPORTED_RB_PROFILE_FRAME_METHOD_NAME' if RUBY_VERSION < '3'
 
+# On older Rubies, there are no Ractors
+$defs << '-DNO_RACTORS' if RUBY_VERSION < '3'
+
+# On older Rubies, rb_imemo_name did not exist
+$defs << '-DNO_IMEMO_NAME' if RUBY_VERSION < '3'
+
+# On older Rubies, objects would not move
+$defs << '-DNO_T_MOVED' if RUBY_VERSION < '2.7'
+
+# On older Rubies, rb_global_vm_lock_struct did not include the owner field
+$defs << '-DNO_GVL_OWNER' if RUBY_VERSION < '2.6'
+
+# On older Rubies, there was no thread->invoke_arg
+$defs << '-DNO_THREAD_INVOKE_ARG' if RUBY_VERSION < '2.6'
+
 # On older Rubies, we need to use rb_thread_t instead of rb_execution_context_t
 $defs << '-DUSE_THREAD_INSTEAD_OF_EXECUTION_CONTEXT' if RUBY_VERSION < '2.5'
 
+# On older Rubies, extensions can't use GET_VM()
+$defs << '-DNO_GET_VM' if RUBY_VERSION < '2.5'
+
 # On older Rubies...
 if RUBY_VERSION < '2.4'
   # ...we need to use RUBY_VM_NORMAL_ISEQ_P instead of VM_FRAME_RUBYFRAME_P
@@ -136,23 +192,22 @@ if RUBY_VERSION < '2.4'
   $defs << '-DUSE_LEGACY_RB_VM_FRAME_METHOD_ENTRY'
 end
 
-# For REALLY OLD Rubies...
-if RUBY_VERSION < '2.3'
-  # ...there was no rb_time_timespec_new function
-  $defs << '-DNO_RB_TIME_TIMESPEC_NEW'
-  # ...the VM changed enough that we need an alternative legacy rb_profile_frames
-  $defs << '-DUSE_LEGACY_RB_PROFILE_FRAMES'
-  # ... you couldn't name threads
-  $defs << '-DNO_THREAD_NAMES'
-  # ...the ruby_thread_has_gvl_p function was not exposed to users outside of the VM
-  $defs << '-DNO_THREAD_HAS_GVL'
-end
+# On older Rubies, rb_gc_force_recycle allowed to free objects in a way that
+# would be invisible to free tracepoints, finalizers and without cleaning
+# obj_to_id_tbl mappings.
+$defs << '-DHAVE_WORKING_RB_GC_FORCE_RECYCLE' if RUBY_VERSION < '3.1'
+
+# On older Rubies, there was no RUBY_SEEN_OBJ_ID flag
+$defs << '-DNO_SEEN_OBJ_ID_FLAG' if RUBY_VERSION < '2.7'
 
 # If we got here, libdatadog is available and loaded
 ENV['PKG_CONFIG_PATH'] = "#{ENV['PKG_CONFIG_PATH']}:#{Libdatadog.pkgconfig_folder}"
-Logging.message(" [ddtrace] PKG_CONFIG_PATH set to #{ENV['PKG_CONFIG_PATH'].inspect}\n")
+Logging.message("[ddtrace] PKG_CONFIG_PATH set to #{ENV['PKG_CONFIG_PATH'].inspect}\n")
+$stderr.puts("Using libdatadog #{Libdatadog::VERSION} from #{Libdatadog.pkgconfig_folder}")
+
+unless pkg_config('datadog_profiling_with_rpath')
+  Logging.message("[ddtrace] Ruby detected the pkg-config command is #{$PKGCONFIG.inspect}\n")
 
-unless pkg_config('ddprof_ffi_with_rpath')
   skip_building_extension!(
     if Datadog::Profiling::NativeExtensionHelpers::Supported.pkg_config_missing?
       Datadog::Profiling::NativeExtensionHelpers::Supported::PKG_CONFIG_IS_MISSING
@@ -163,17 +218,17 @@ unless pkg_config('ddprof_ffi_with_rpath')
   )
 end
 
+unless have_type('atomic_int', ['stdatomic.h'])
+  skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::COMPILER_ATOMIC_MISSING)
+end
+
 # See comments on the helper method being used for why we need to additionally set this.
 # The extremely excessive escaping around ORIGIN below seems to be correct and was determined after a lot of
 # experimentation. We need to get these special characters across a lot of tools untouched...
 $LDFLAGS += \
   ' -Wl,-rpath,$$$\\\\{ORIGIN\\}/' \
   "#{Datadog::Profiling::NativeExtensionHelpers.libdatadog_folder_relative_to_native_lib_folder}"
-Logging.message(" [ddtrace] After pkg-config $LDFLAGS were set to: #{$LDFLAGS.inspect}\n")
-
-Logging.message(" [ddtrace] Using compiler:\n")
-xsystem("#{CONFIG['CC']} --version")
-Logging.message(" [ddtrace] End of compiler information\n")
+Logging.message("[ddtrace] After pkg-config $LDFLAGS were set to: #{$LDFLAGS.inspect}\n")
 
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
@@ -202,9 +257,14 @@ if Datadog::Profiling::NativeExtensionHelpers::CAN_USE_MJIT_HEADER
   # NOTE: This needs to come after all changes to $defs
   create_header
 
+  # Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+  # See the comment on the same flag below for why this is done last.
+  add_compiler_flag '-Wunused-parameter'
+
   create_makefile EXTENSION_NAME
 else
-  # On older Rubies, we use the debase-ruby_core_source gem to get access to private VM headers.
+  # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
+  # the debase-ruby_core_source gem to get access to private VM headers.
   # This gem ships source code copies of these VM headers for the different Ruby VM versions;
   # see https://github.com/ruby-debug/debase-ruby_core_source for details
 
@@ -215,7 +275,22 @@ else
 
   Debase::RubyCoreSource
     .create_makefile_with_core(
-      proc { have_header('vm_core.h') && have_header('iseq.h') },
+      proc do
+        headers_available =
+          have_header('vm_core.h') &&
+          have_header('iseq.h') &&
+          (RUBY_VERSION < '3.3' || have_header('ractor_core.h'))
+
+        if headers_available
+          # Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+          # This is added as late as possible because in some Rubies we support (e.g. 3.3), adding this flag before
+          # checking if internal VM headers are available causes those checks to fail because of this warning (and not
+          # because the headers are not available.)
+          add_compiler_flag '-Wunused-parameter'
+        end
+
+        headers_available
+      end,
       EXTENSION_NAME,
     )
 end