ddtrace 0.54.1 → 0.54.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bef2da14245890c8187ebffd9cce972da89a177a806c6991f2e3050e1de62bc3
-  data.tar.gz: d37dcaf645e16177c6f7a3dd77628f2e5a4920c45fca8e3f78665c34a4d76db2
+  metadata.gz: 000f0860f13e54bd5d4e0e2d0c4803eb8c568e808379f6ad58b9c9132c03562e
+  data.tar.gz: 26433871947470f4ba39de8b0790385a7dd3e7d8ff6e04aa28aff717ec81a490
 SHA512:
-  metadata.gz: c59d7c5b2f363eb96f78ad602489ccbd0ed94ae9d6e856c70272ddb21ba467255179814db1831518789b7c32e9ab5d0f13421dd67ea585d0b39fe78849bcfe81
-  data.tar.gz: 97dcba8ee6eda7c3cafdb39946863d337ef67843ac7f071890ce6253786b9d9a26ca8a9f181a8c0dad44c01a3ffab53c146ea4848b6497552d26558f9f1278ce
+  metadata.gz: 4f2282c9f65efc83c4b815f64f51cb352adb443ca059b6fbeed921a8db83bd0f076060b190ff2a41e024a77599e80b7e2fdbc0a53c0f701a799f0541a53a64cc
+  data.tar.gz: aa4c8ef2ab73071bd1e58c367d300799483aac3c79e1fee3483bc917316da1cd49fae8a999f4a2c0dfc4a7b081478bd872ca2b6fbfe812f07f6bfa7329de304b

data/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## [Unreleased]
 
+## [0.54.2] - 2022-01-18
+
+Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.54.2
+
+Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.54.1...v0.54.2
+
+### Changed
+
+- Bump `debase-ruby_core_source` dependency version; also allow older versions to be used ([#1798][], [#1829][])
+- Profiler: Reduce impact of reporting data in multi-process applications ([#1807][])
+- Profiler: Update API used to report data to backend ([#1820][])
+
+### Fixed
+
+- Gracefully handle installation on environments where Ruby JIT seems to be available but is actually broken ([#1801][])
+
 ## [0.54.1] - 2021-11-30
 
 Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.54.1
@@ -2682,6 +2698,11 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#1771]: https://github.com/DataDog/dd-trace-rb/issues/1771
 [#1774]: https://github.com/DataDog/dd-trace-rb/issues/1774
 [#1776]: https://github.com/DataDog/dd-trace-rb/issues/1776
+[#1798]: https://github.com/DataDog/dd-trace-rb/issues/1798
+[#1801]: https://github.com/DataDog/dd-trace-rb/issues/1801
+[#1807]: https://github.com/DataDog/dd-trace-rb/issues/1807
+[#1820]: https://github.com/DataDog/dd-trace-rb/issues/1820
+[#1829]: https://github.com/DataDog/dd-trace-rb/issues/1829
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot

data/ddtrace.gemspec

@@ -50,7 +50,7 @@ Gem::Specification.new do |spec|
   #
   # Because we only use this for older Rubies, and we consider it "feature-complete" for those older Rubies,
   # we're pinning it at the latest available version and will manually bump the dependency as needed.
-  spec.add_dependency 'debase-ruby_core_source', '= 0.10.12'
+  spec.add_dependency 'debase-ruby_core_source', '<= 0.10.14'
 
   spec.extensions = ['ext/ddtrace_profiling_native_extension/extconf.rb']
 end

data/docs/GettingStarted.md

@@ -135,9 +135,13 @@ To contribute, check out the [contribution guidelines][contribution docs] and [d
 
 *EOL* indicates support is no longer provided.
 
+### Apple macOS support
+
+Use of `ddtrace` on macOS is supported for development, but not for production deployments.
+
 ### Microsoft Windows support
 
-Using `ddtrace ` on Microsoft Windows is currently unsupported. We'll still accept community contributions and issues,
+Using `ddtrace` on Microsoft Windows is currently unsupported. We'll still accept community contributions and issues,
 but will consider them as having low priority.
 
 ## Installation

data/ext/ddtrace_profiling_native_extension/extconf.rb

@@ -1,4 +1,5 @@
 # typed: ignore
+# rubocop:disable Style/StderrPuts
 
 # Older Rubies don't have the MJIT header, used by the JIT compiler, so we need to use a different approach
 CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
@@ -26,7 +27,6 @@ def expected_to_use_mjit_but_mjit_is_disabled?
   mjit_disabled = CAN_USE_MJIT_HEADER && RbConfig::CONFIG['MJIT_SUPPORT'] != 'yes'
 
   if mjit_disabled
-    # rubocop:disable Style/StderrPuts
     $stderr.puts(%(
 +------------------------------------------------------------------------------+
 | Your Ruby has been compiled without JIT support (--disable-jit-support).     |
@@ -55,11 +55,16 @@ def skip_building_extension?
 end
 
 # IMPORTANT: When adding flags, remember that our customers compile with a wide range of gcc/clang versions, so
-# doublecheck that what you're adding can be reasonably expected to exist on their systems.
+# doublecheck that what you're adding can be reasonably expected to work on their systems.
 def add_compiler_flag(flag)
   $CFLAGS << ' ' << flag
 end
 
+def skip_building_extension!
+  File.write('Makefile', 'all install clean: # dummy makefile that does nothing')
+  exit
+end
+
 if skip_building_extension?
   $stderr.puts(%(
 +------------------------------------------------------------------------------+
@@ -68,14 +73,12 @@ if skip_building_extension?
 +------------------------------------------------------------------------------+
 
 ))
-
-  File.write('Makefile', 'all install clean: # dummy makefile that does nothing')
-  return
+  skip_building_extension!
 end
 
 $stderr.puts(%(
 +------------------------------------------------------------------------------+
-| **Preparing to build the ddtrace native extension...**                       |
+| ** Preparing to build the ddtrace native extension... **                     |
 |                                                                              |
 | If you run into any failures during this step, you can set the               |
 | `DD_PROFILING_NO_EXTENSION` environment variable to `true` e.g.              |
@@ -90,7 +93,6 @@ $stderr.puts(%(
 +------------------------------------------------------------------------------+
 
 ))
-# rubocop:enable Style/StderrPuts
 
 # NOTE: we MUST NOT require 'mkmf' before we check the #skip_building_extension? because the require triggers checks
 # that may fail on an environment not properly setup for building Ruby extensions.
@@ -124,6 +126,34 @@ end
 EXTENSION_NAME = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}".freeze
 
 if CAN_USE_MJIT_HEADER
+  mjit_header_file_name = "rb_mjit_min_header-#{RUBY_VERSION}.h"
+
+  # Validate that the mjit header can actually be compiled on this system. We learned via
+  # https://github.com/DataDog/dd-trace-rb/issues/1799 and https://github.com/DataDog/dd-trace-rb/issues/1792
+  # that even if the header seems to exist, it may not even compile.
+  # `have_macro` actually tries to compile a file that mentions the given macro, so if this passes, we should be good to
+  # use the MJIT header.
+  # Finally, the `COMMON_HEADERS` conflict with the MJIT header so we need to temporarily disable them for this check.
+  original_common_headers = MakeMakefile::COMMON_HEADERS
+  MakeMakefile::COMMON_HEADERS = ''.freeze
+  unless have_macro('RUBY_MJIT_H', mjit_header_file_name)
+    $stderr.puts(%(
++------------------------------------------------------------------------------+
+| WARNING: Unable to compile a needed component for ddtrace native extension.  |
+| Your C compiler or Ruby VM just-in-time compiler seems to be broken.         |
+|                                                                              |
+| You will be NOT be able to use ddtrace profiling features,                   |
+| but all other features will work fine!                                       |
+|                                                                              |
+| For help solving this issue, please contact Datadog support at               |
+| <https://docs.datadoghq.com/help/>.                                          |
++------------------------------------------------------------------------------+
+
+))
+    skip_building_extension!
+  end
+  MakeMakefile::COMMON_HEADERS = original_common_headers
+
   $defs << '-DUSE_MJIT_HEADER'
 
   # NOTE: This needs to come after all changes to $defs
@@ -136,7 +166,7 @@ if CAN_USE_MJIT_HEADER
     File.read($extconf_h)
         .sub('#endif',
              <<-EXTCONF_H.strip
-#define RUBY_MJIT_HEADER "rb_mjit_min_header-#{RUBY_VERSION}.h"
+#define RUBY_MJIT_HEADER "#{mjit_header_file_name}"
 
 #endif
              EXTCONF_H
@@ -164,3 +194,4 @@ else
   Debase::RubyCoreSource
     .create_makefile_with_core(proc { have_header('vm_core.h') && thread_native_for_ruby_2_1.call }, EXTENSION_NAME)
 end
+# rubocop:enable Style/StderrPuts

data/lib/ddtrace/ext/profiling.rb

@@ -23,13 +23,9 @@ module Datadog
         module HTTP
           URI_TEMPLATE_DD_API = 'https://intake.profile.%s/'.freeze
 
-          FORM_FIELD_DATA = 'data[0]'.freeze
-          FORM_FIELD_FORMAT = 'format'.freeze
-          FORM_FIELD_FORMAT_PPROF = 'pprof'.freeze
-          FORM_FIELD_RECORDING_END = 'recording-end'.freeze
-          FORM_FIELD_RECORDING_START = 'recording-start'.freeze
-          FORM_FIELD_RUNTIME = 'runtime'.freeze
-          FORM_FIELD_RUNTIME_ID = 'runtime-id'.freeze
+          FORM_FIELD_RECORDING_START = 'start'.freeze
+          FORM_FIELD_RECORDING_END = 'end'.freeze
+          FORM_FIELD_FAMILY = 'family'.freeze
           FORM_FIELD_TAG_ENV = 'env'.freeze
           FORM_FIELD_TAG_HOST = 'host'.freeze
           FORM_FIELD_TAG_LANGUAGE = 'language'.freeze
@@ -43,13 +39,13 @@ module Datadog
           FORM_FIELD_TAG_SERVICE = 'service'.freeze
           FORM_FIELD_TAG_VERSION = 'version'.freeze
           FORM_FIELD_TAGS = 'tags'.freeze
-          FORM_FIELD_TYPES = 'types[0]'.freeze
-          FORM_FIELD_TYPES_AUTO = 'auto'.freeze
+          FORM_FIELD_INTAKE_VERSION = 'version'.freeze
 
           HEADER_CONTENT_TYPE = 'Content-Type'.freeze
           HEADER_CONTENT_TYPE_OCTET_STREAM = 'application/octet-stream'.freeze
 
-          PPROF_DEFAULT_FILENAME = 'profile.pb.gz'.freeze
+          FORM_FIELD_PPROF_DATA = 'data[rubyprofile.pprof]'.freeze
+          PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof.gz'.freeze
         end
       end
     end

data/lib/ddtrace/profiling/collectors/stack.rb

@@ -88,10 +88,6 @@ module Datadog
           collect_and_wait
         end
 
-        def loop_back_off?
-          false
-        end
-
         def collect_and_wait
           run_time = Datadog::Utils::Time.measure do
             collect_events

data/lib/ddtrace/profiling/scheduler.rb

@@ -17,6 +17,10 @@ module Datadog
       # Profiles with duration less than this will not be reported
       PROFILE_DURATION_THRESHOLD_SECONDS = 1
 
+      # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
+      # reporting profiles at the exact same time
+      DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
+
       private_constant :DEFAULT_INTERVAL_SECONDS, :MINIMUM_INTERVAL_SECONDS, :PROFILE_DURATION_THRESHOLD_SECONDS
 
       attr_reader \
@@ -62,10 +66,6 @@ module Datadog
         end
       end
 
-      def loop_back_off?
-        false
-      end
-
       def after_fork
         # Clear recorder's buffers by flushing events.
         # Objects from parent process will copy-on-write,
@@ -110,6 +110,22 @@ module Datadog
           return flush
         end
 
+        # Sleep for a bit to cause misalignment between profilers in multi-process applications
+        #
+        # When not being run in a loop, it means the scheduler has not been started or was stopped, and thus
+        # a) it's being shutting down (and is trying to report the last profile)
+        # b) it's being run as a one-shot, usually in a test
+        # ...so in those cases we don't sleep
+        #
+        # During PR review (https://github.com/DataDog/dd-trace-rb/pull/1807) we discussed the possible alternative of
+        # just sleeping before starting the scheduler loop. We ended up not going with that option to avoid the first
+        # profile containing up to DEFAULT_INTERVAL_SECONDS + DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS instead of the
+        # usual DEFAULT_INTERVAL_SECONDS size.
+        if run_loop?
+          jitter_seconds = rand * DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS # floating point number between (0.0...maximum)
+          sleep(jitter_seconds)
+        end
+
         # Send events to each exporter
         if flush.event_count > 0
           exporters.each do |exporter|

data/lib/ddtrace/profiling/transport/http/api/endpoint.rb

@@ -41,11 +41,10 @@ module Datadog
 
             def build_form(env)
               flush = env.request.parcel.data
-              pprof_file, types = build_pprof(flush)
+              pprof_file = build_pprof(flush)
 
               form = {
-                # NOTE: Redundant w/ 'runtime-id' tag below; may want to remove this later.
-                FORM_FIELD_RUNTIME_ID => flush.runtime_id,
+                FORM_FIELD_INTAKE_VERSION => '3', # Aka 1.3 intake format
                 FORM_FIELD_RECORDING_START => flush.start.utc.iso8601,
                 FORM_FIELD_RECORDING_END => flush.finish.utc.iso8601,
                 FORM_FIELD_TAGS => [
@@ -64,14 +63,10 @@ module Datadog
                     .reject { |tag_key| TAGS_TO_IGNORE_IN_TAGS_HASH.include?(tag_key) }
                     .map { |tag_key, tag_value| "#{tag_key}:#{tag_value}" }
                 ],
-                FORM_FIELD_DATA => pprof_file,
-                FORM_FIELD_RUNTIME => flush.language,
-                FORM_FIELD_FORMAT => FORM_FIELD_FORMAT_PPROF
+                FORM_FIELD_PPROF_DATA => pprof_file,
+                FORM_FIELD_FAMILY => flush.language,
               }
 
-              # Add types
-              form[FORM_FIELD_TYPES] = types.join(',')
-
               # Optional fields
               form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_SERVICE}:#{flush.service}" unless flush.service.nil?
               form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_ENV}:#{flush.env}" unless flush.env.nil?
@@ -83,15 +78,13 @@ module Datadog
             def build_pprof(flush)
               pprof = encoder.encode(flush)
 
-              # Wrap pprof as a gzipped file
-              gzipped_data = Datadog::Utils::Compression.gzip(pprof.data)
-              pprof_file = Datadog::Vendor::Multipart::Post::UploadIO.new(
-                StringIO.new(gzipped_data),
+              gzipped_pprof_data = Datadog::Utils::Compression.gzip(pprof.data)
+
+              Datadog::Vendor::Multipart::Post::UploadIO.new(
+                StringIO.new(gzipped_pprof_data),
                 HEADER_CONTENT_TYPE_OCTET_STREAM,
                 PPROF_DEFAULT_FILENAME
               )
-
-              [pprof_file, [FORM_FIELD_TYPES_AUTO]]
             end
           end
         end

data/lib/ddtrace/version.rb

@@ -3,7 +3,7 @@ module Datadog
   module VERSION
     MAJOR = 0
     MINOR = 54
-    PATCH = 1
+    PATCH = 2
     PRE = nil
 
     STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')

data/lib/ddtrace/workers/{loop.rb → interval_loop.rb}

@@ -60,15 +60,6 @@ module Datadog
         @loop_wait_time = value
       end
 
-      def reset_loop_wait_time
-        self.loop_wait_time = loop_base_interval
-      end
-
-      # Should the loop "back off" when there's no work?
-      def loop_back_off?
-        false
-      end
-
       def loop_back_off!
         self.loop_wait_time = [loop_wait_time * BACK_OFF_RATIO, BACK_OFF_MAX].min
       end
@@ -106,13 +97,6 @@ module Datadog
             # There's work to do...
             # Run the task
             yield
-
-            # Reset the wait interval
-            reset_loop_wait_time if loop_back_off?
-          elsif loop_back_off?
-            # There's no work to do...
-            # Back off the wait interval a bit
-            loop_back_off!
           end
 
           # Wait for an interval, unless shutdown has been signaled.

data/lib/ddtrace/workers/polling.rb

@@ -1,6 +1,6 @@
 # typed: false
 require 'ddtrace/workers/async'
-require 'ddtrace/workers/loop'
+require 'ddtrace/workers/interval_loop'
 
 module Datadog
   module Workers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddtrace
 version: !ruby/object:Gem::Version
-  version: 0.54.1
+  version: 0.54.2
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-01 00:00:00.000000000 Z
+date: 2022-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -28,16 +28,16 @@ dependencies:
   name: debase-ruby_core_source
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 0.10.12
+        version: 0.10.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 0.10.12
+        version: 0.10.14
 description: |
   ddtrace is Datadog’s tracing client for Ruby. It is used to trace requests
   as they flow across web servers, databases and microservices so that developers
@@ -611,7 +611,7 @@ files:
 - lib/ddtrace/worker.rb
 - lib/ddtrace/workers.rb
 - lib/ddtrace/workers/async.rb
-- lib/ddtrace/workers/loop.rb
+- lib/ddtrace/workers/interval_loop.rb
 - lib/ddtrace/workers/polling.rb
 - lib/ddtrace/workers/queue.rb
 - lib/ddtrace/workers/runtime_metrics.rb