ddtrace 0.54.0 → 1.1.0

data/ext/ddtrace_profiling_native_extension/extconf.rb

@@ -1,55 +1,40 @@
 # typed: ignore
 
-def skip_building_extension?
-  # We don't support JRuby for profiling, and JRuby doesn't support native extensions, so let's just skip this entire
-  # thing so that JRuby users of dd-trace-rb aren't impacted.
-  on_jruby = RUBY_ENGINE == 'jruby'
+# rubocop:disable Style/StderrPuts
+# rubocop:disable Style/GlobalVars
 
-  # We don't officially support TruffleRuby for dd-trace-rb at all BUT let's not break adventurous customers that
-  # want to give it a try.
-  on_truffleruby = RUBY_ENGINE == 'truffleruby'
+require_relative 'native_extension_helpers'
 
-  # Microsoft Windows is unsupported, so let's not build the extension there.
-  on_windows = Gem.win_platform?
+SKIPPED_REASON_FILE = "#{__dir__}/skipped_reason.txt".freeze
+# Not a problem if the file doesn't exist or we can't delete it
+File.delete(SKIPPED_REASON_FILE) rescue nil
 
-  # Experimental toggle to disable building the extension.
-  # Disabling the extension will lead to the profiler not working in future releases.
-  # If you needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.
-  disabled_via_env = ENV['DD_PROFILING_NO_EXTENSION'].to_s.downcase == 'true'
+def skip_building_extension!(reason)
+  $stderr.puts(Datadog::Profiling::NativeExtensionHelpers::Supported.failure_banner_for(**reason))
+  File.write(
+    SKIPPED_REASON_FILE,
+    Datadog::Profiling::NativeExtensionHelpers::Supported.render_skipped_reason_file(**reason),
+  )
 
-  on_jruby || on_truffleruby || on_windows || disabled_via_env
-end
-
-# IMPORTANT: When adding flags, remember that our customers compile with a wide range of gcc/clang versions, so
-# doublecheck that what you're adding can be reasonably expected to exist on their systems.
-def add_compiler_flag(flag)
-  $CFLAGS << ' ' << flag
+  File.write('Makefile', 'all install clean: # dummy makefile that does nothing')
+  exit
 end
 
-if skip_building_extension?
-  # rubocop:disable Style/StderrPuts
-  $stderr.puts(%(
-+------------------------------------------------------------------------------+
-| Skipping build of profiling native extension and replacing it with a no-op   |
-| Makefile                                                                     |
-+------------------------------------------------------------------------------+
-
-))
-
-  File.write('Makefile', 'all install clean: # dummy makefile that does nothing')
-  return
+unless Datadog::Profiling::NativeExtensionHelpers::Supported.supported?
+  skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported.unsupported_reason)
 end
 
 $stderr.puts(%(
 +------------------------------------------------------------------------------+
-| **Preparing to build the ddtrace native extension...**                       |
+| ** Preparing to build the ddtrace profiling native extension... **           |
 |                                                                              |
 | If you run into any failures during this step, you can set the               |
 | `DD_PROFILING_NO_EXTENSION` environment variable to `true` e.g.              |
 | `$ DD_PROFILING_NO_EXTENSION=true bundle install` to skip this step.         |
 |                                                                              |
-| Disabling the extension will lead to the ddtrace profiling features not      |
-| working in future releases.                                                  |
+| If you disable this extension, the Datadog Continuous Profiler will          |
+| not be available, but all other ddtrace features will work fine!             |
+|                                                                              |
 | If you needed to use this, please tell us why on                             |
 | <https://github.com/DataDog/dd-trace-rb/issues/new> so we can fix it :\)      |
 |                                                                              |
@@ -57,12 +42,27 @@ $stderr.puts(%(
 +------------------------------------------------------------------------------+
 
 ))
-# rubocop:enable Style/StderrPuts
 
 # NOTE: we MUST NOT require 'mkmf' before we check the #skip_building_extension? because the require triggers checks
 # that may fail on an environment not properly setup for building Ruby extensions.
 require 'mkmf'
 
+# mkmf on modern Rubies actually has an append_cflags that does something similar
+# (see https://github.com/ruby/ruby/pull/5760), but as usual we need a bit more boilerplate to deal with legacy Rubies
+def add_compiler_flag(flag)
+  if try_cflags(flag)
+    $CFLAGS << ' ' << flag
+  else
+    $stderr.puts("WARNING: '#{flag}' not accepted by compiler, skipping it")
+  end
+end
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another ddtrace dependency
+#   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
+add_compiler_flag '-std=gnu99'
+
 # Gets really noisy when we include the MJIT header, let's omit it
 add_compiler_flag '-Wno-unused-function'
 
@@ -73,6 +73,14 @@ add_compiler_flag '-Wno-declaration-after-statement'
 # cause a segfault later. Let's ensure that never happens.
 add_compiler_flag '-Werror-implicit-function-declaration'
 
+# The native extension is not intended to expose any symbols/functions for other native libraries to use;
+# the sole exception being `Init_ddtrace_profiling_native_extension` which needs to be visible for Ruby to call it when
+# it `dlopen`s the library.
+#
+# By setting this compiler flag, we tell it to assume that everything is private unless explicitly stated.
+# For more details see https://gcc.gnu.org/wiki/Visibility
+add_compiler_flag '-fvisibility=hidden'
+
 if RUBY_PLATFORM.include?('linux')
   # Supposedly, the correct way to do this is
   # ```
@@ -84,8 +92,33 @@ if RUBY_PLATFORM.include?('linux')
   $defs << '-DHAVE_PTHREAD_GETCPUCLOCKID'
 end
 
-# Older Rubies don't have the MJIT header, used by the JIT compiler, so we need to use a different approach
-CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
+# On older Rubies, we need to use a backported version of this function. See private_vm_api_access.h for details.
+$defs << '-DUSE_BACKPORTED_RB_PROFILE_FRAME_METHOD_NAME' if RUBY_VERSION < '3'
+
+# On older Rubies, we need to use rb_thread_t instead of rb_execution_context_t
+$defs << '-DUSE_THREAD_INSTEAD_OF_EXECUTION_CONTEXT' if RUBY_VERSION < '2.5'
+
+# On older Rubies...
+if RUBY_VERSION < '2.4'
+  # ...we need to use RUBY_VM_NORMAL_ISEQ_P instead of VM_FRAME_RUBYFRAME_P
+  $defs << '-DUSE_ISEQ_P_INSTEAD_OF_RUBYFRAME_P'
+  # ...we use a legacy copy of rb_vm_frame_method_entry
+  $defs << '-DUSE_LEGACY_RB_VM_FRAME_METHOD_ENTRY'
+end
+
+# For REALLY OLD Rubies...
+if RUBY_VERSION < '2.3'
+  # ...there was no rb_time_timespec_new function
+  $defs << '-DNO_RB_TIME_TIMESPEC_NEW'
+  # ...the VM changed enough that we need an alternative legacy rb_profile_frames
+  $defs << '-DUSE_LEGACY_RB_PROFILE_FRAMES'
+end
+
+# If we got here, libddprof is available and loaded
+ENV['PKG_CONFIG_PATH'] = "#{ENV['PKG_CONFIG_PATH']}:#{Libddprof.pkgconfig_folder}"
+unless pkg_config('ddprof_ffi_with_rpath')
+  skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::FAILED_TO_CONFIGURE_LIBDDPROF)
+end
 
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
@@ -93,26 +126,27 @@ CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
 # When requiring, we need to use the exact same string, including the version and the platform.
 EXTENSION_NAME = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}".freeze
 
-if CAN_USE_MJIT_HEADER
-  $defs << '-DUSE_MJIT_HEADER'
+if Datadog::Profiling::NativeExtensionHelpers::CAN_USE_MJIT_HEADER
+  mjit_header_file_name = "rb_mjit_min_header-#{RUBY_VERSION}.h"
+
+  # Validate that the mjit header can actually be compiled on this system. We learned via
+  # https://github.com/DataDog/dd-trace-rb/issues/1799 and https://github.com/DataDog/dd-trace-rb/issues/1792
+  # that even if the header seems to exist, it may not even compile.
+  # `have_macro` actually tries to compile a file that mentions the given macro, so if this passes, we should be good to
+  # use the MJIT header.
+  # Finally, the `COMMON_HEADERS` conflict with the MJIT header so we need to temporarily disable them for this check.
+  original_common_headers = MakeMakefile::COMMON_HEADERS
+  MakeMakefile::COMMON_HEADERS = ''.freeze
+  unless have_macro('RUBY_MJIT_H', mjit_header_file_name)
+    skip_building_extension!(Datadog::Profiling::NativeExtensionHelpers::Supported::COMPILATION_BROKEN)
+  end
+  MakeMakefile::COMMON_HEADERS = original_common_headers
+
+  $defs << "-DRUBY_MJIT_HEADER='\"#{mjit_header_file_name}\"'"
 
   # NOTE: This needs to come after all changes to $defs
   create_header
 
-  # The MJIT header is always (afaik?) suffixed with the exact Ruby VM version,
-  # including patch (e.g. 2.7.2). Thus, we add to the header file a definition
-  # containing the exact file, so that it can be used in a #include in the C code.
-  header_contents =
-    File.read($extconf_h)
-        .sub('#endif',
-             <<-EXTCONF_H.strip
-#define RUBY_MJIT_HEADER "rb_mjit_min_header-#{RUBY_VERSION}.h"
-
-#endif
-             EXTCONF_H
-            )
-  File.open($extconf_h, 'w') { |file| file.puts(header_contents) }
-
   create_makefile EXTENSION_NAME
 else
   # On older Rubies, we use the debase-ruby_core_source gem to get access to private VM headers.
@@ -132,5 +166,11 @@ else
   dir_config('ruby') # allow user to pass in non-standard core include directory
 
   Debase::RubyCoreSource
-    .create_makefile_with_core(proc { have_header('vm_core.h') && thread_native_for_ruby_2_1.call }, EXTENSION_NAME)
+    .create_makefile_with_core(
+      proc { have_header('vm_core.h') && have_header('iseq.h') && thread_native_for_ruby_2_1.call },
+      EXTENSION_NAME,
+    )
 end
+
+# rubocop:enable Style/GlobalVars
+# rubocop:enable Style/StderrPuts

data/ext/ddtrace_profiling_native_extension/libddprof_helpers.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <ddprof/ffi.h>
+
+inline static ddprof_ffi_CharSlice char_slice_from_ruby_string(VALUE string) {
+  Check_Type(string, T_STRING);
+  ddprof_ffi_CharSlice char_slice = {.ptr = StringValuePtr(string), .len = RSTRING_LEN(string)};
+  return char_slice;
+}
+
+inline static VALUE ruby_string_from_vec_u8(ddprof_ffi_Vec_u8 string) {
+  return rb_str_new((char *) string.ptr, string.len);
+}

data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+# typed: ignore
+
+require 'libddprof'
+
+module Datadog
+  module Profiling
+    module NativeExtensionHelpers
+      ENV_NO_EXTENSION = 'DD_PROFILING_NO_EXTENSION'
+
+      # Older Rubies don't have the MJIT header, used by the JIT compiler, so we need to use a different approach
+      CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
+
+      # Used to check if profiler is supported, including user-visible clear messages explaining why their
+      # system may not be supported.
+      # rubocop:disable Metrics/ModuleLength
+      module Supported
+        private_class_method def self.explain_issue(*reason, suggested:)
+          { reason: reason, suggested: suggested }
+        end
+
+        def self.supported?
+          unsupported_reason.nil?
+        end
+
+        def self.unsupported_reason
+          disabled_via_env? ||
+            on_jruby? ||
+            on_truffleruby? ||
+            on_windows? ||
+            on_macos? ||
+            on_unknown_os? ||
+            not_on_amd64_or_arm64? ||
+            expected_to_use_mjit_but_mjit_is_disabled? ||
+            libddprof_not_usable?
+        end
+
+        # This banner will show up in the logs/terminal while compiling the native extension
+        def self.failure_banner_for(reason:, suggested:)
+          prettify_lines = proc { |lines| lines.map { |line| "| #{line.ljust(76)} |" }.join("\n") }
+          %(
++------------------------------------------------------------------------------+
+| Could not compile the Datadog Continuous Profiler because                    |
+#{prettify_lines.call(reason)}
+|                                                                              |
+| The Datadog Continuous Profiler will not be available,                       |
+| but all other ddtrace features will work fine!                               |
+|                                                                              |
+#{prettify_lines.call(suggested)}
++------------------------------------------------------------------------------+
+          )
+        end
+
+        # This will be saved in a file to later be presented while operating the gem
+        def self.render_skipped_reason_file(reason:, suggested:)
+          [*reason, *suggested].join(' ')
+        end
+
+        CONTACT_SUPPORT = [
+          'For help solving this issue, please contact Datadog support at',
+          '<https://docs.datadoghq.com/help/>.',
+        ].freeze
+
+        REPORT_ISSUE = [
+          'If you needed to use this, please tell us why on',
+          '<https://github.com/DataDog/dd-trace-rb/issues/new> so we can fix it :)',
+        ].freeze
+
+        GET_IN_TOUCH = [
+          "Get in touch with us if you're interested in profiling your app!"
+        ].freeze
+
+        # Validation for this check is done in extconf.rb because it relies on mkmf
+        FAILED_TO_CONFIGURE_LIBDDPROF = explain_issue(
+          'there was a problem in setting up the `libddprof` dependency.',
+          suggested: CONTACT_SUPPORT,
+        )
+
+        # Validation for this check is done in extconf.rb because it relies on mkmf
+        COMPILATION_BROKEN = explain_issue(
+          'compilation of the Ruby VM just-in-time header failed.',
+          'Your C compiler or Ruby VM just-in-time compiler seem to be broken.',
+          suggested: CONTACT_SUPPORT,
+        )
+
+        private_class_method def self.disabled_via_env?
+          disabled_via_env = explain_issue(
+            'the `DD_PROFILING_NO_EXTENSION` environment variable is/was set to',
+            '`true` during installation.',
+            suggested: REPORT_ISSUE,
+          )
+
+          return unless ENV[ENV_NO_EXTENSION].to_s.strip.downcase == 'true'
+
+          disabled_via_env
+        end
+
+        private_class_method def self.on_jruby?
+          jruby_not_supported = explain_issue(
+            'JRuby is not supported by the Datadog Continuous Profiler.',
+            suggested: GET_IN_TOUCH,
+          )
+
+          jruby_not_supported if RUBY_ENGINE == 'jruby'
+        end
+
+        private_class_method def self.on_truffleruby?
+          truffleruby_not_supported = explain_issue(
+            'TruffleRuby is not supported by the ddtrace gem.',
+            suggested: GET_IN_TOUCH,
+          )
+
+          truffleruby_not_supported if RUBY_ENGINE == 'truffleruby'
+        end
+
+        # See https://docs.datadoghq.com/tracing/setup_overview/setup/ruby/#microsoft-windows-support for current
+        # state of Windows support in ddtrace.
+        private_class_method def self.on_windows?
+          windows_not_supported = explain_issue(
+            'Microsoft Windows is not supported by the Datadog Continuous Profiler.',
+            suggested: GET_IN_TOUCH,
+          )
+
+          windows_not_supported if Gem.win_platform?
+        end
+
+        private_class_method def self.on_macos?
+          macos_not_supported = explain_issue(
+            'macOS is currently not supported by the Datadog Continuous Profiler.',
+            suggested: GET_IN_TOUCH,
+          )
+          # For development only; not supported otherwise
+          macos_testing_override = ENV['DD_PROFILING_MACOS_TESTING'] == 'true'
+
+          macos_not_supported if RUBY_PLATFORM.include?('darwin') && !macos_testing_override
+        end
+
+        private_class_method def self.on_unknown_os?
+          unknown_os_not_supported = explain_issue(
+            'your operating system is not supported by the Datadog Continuous Profiler.',
+            suggested: GET_IN_TOUCH,
+          )
+
+          unknown_os_not_supported unless RUBY_PLATFORM.include?('darwin') || RUBY_PLATFORM.include?('linux')
+        end
+
+        private_class_method def self.not_on_amd64_or_arm64?
+          architecture_not_supported = explain_issue(
+            'your CPU architecture is not supported by the Datadog Continuous Profiler.',
+            suggested: GET_IN_TOUCH,
+          )
+
+          architecture_not_supported unless RUBY_PLATFORM.start_with?('x86_64', 'aarch64')
+        end
+
+        # On some Rubies, we require the mjit header to be present. If Ruby was installed without MJIT support, we also skip
+        # building the extension.
+        private_class_method def self.expected_to_use_mjit_but_mjit_is_disabled?
+          ruby_without_mjit = explain_issue(
+            'your Ruby has been compiled without JIT support (--disable-jit-support).',
+            'The profiling native extension requires a Ruby compiled with JIT support,',
+            'even if the JIT is not in use by the application itself.',
+            suggested: CONTACT_SUPPORT,
+          )
+
+          ruby_without_mjit if CAN_USE_MJIT_HEADER && RbConfig::CONFIG['MJIT_SUPPORT'] != 'yes'
+        end
+
+        private_class_method def self.libddprof_not_usable?
+          no_binaries_for_current_platform = explain_issue(
+            'the `libddprof` gem installed on your system is missing binaries for your',
+            'platform variant.',
+            "(Your platform: `#{Gem::Platform.local}`)",
+            '(Available binaries: ',
+            "`#{Libddprof.available_binaries.join('`, `')}`)",
+            suggested: CONTACT_SUPPORT,
+          )
+
+          no_binaries_for_current_platform unless Libddprof.pkgconfig_folder
+        end
+      end
+      # rubocop:enable Metrics/ModuleLength
+    end
+  end
+end