ddtrace 1.4.2 → 1.19.0

data/LICENSE-3rdparty.csv

@@ -1,6 +1,8 @@
 Component,Origin,License,Copyright
 lib/datadog/core/vendor/multipart-post,https://github.com/socketry/multipart-post,MIT,"Copyright (c) 2007-2013 Nick Sieger."
 lib/datadog/tracing/contrib/active_record/vendor,https://github.com/rails/rails/,MIT,"Copyright (c) 2005-2018 David Heinemeier Hansson"
+lib/datadog/tracing/contrib/utils/quantization/http.rb,https://github.com/ruby/uri,BSD-2-Clause,"Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
 ext/ddtrace_profiling_native_extension/private_vm_api_access,https://github.com/ruby/ruby,BSD-2-Clause,"Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
 msgpack,https://rubygems.org/gems/msgpack,Apache-2.0,"Copyright (c) 2008-2015 Sadayuki Furuhashi"
 debase-ruby_core_source,https://rubygems.org/gems/debase-ruby_core_source,MIT for gem and BSD-2-Clause for Ruby sources,"Copyright (c) 2012 Gabriel Horner. Files from Ruby sources are Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
+lib/datadog/core/vendor/ipaddr,https://github.com/ruby/ipaddr/blob/master/lib/ipaddr.rb,BSD 2-Clause "Simplified" License,"Copyright (c) 2002 Hajimu UMEMOTO <ume@mahoroba.org> Copyright (c) 2007-2017 Akinori MUSHA <knu@iDaemons.org>"

data/README.md

@@ -1,29 +1,27 @@
-**We've recently released the 1.x version series. If you're upgrading from a 0.x version, check out our [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md#from-0x-to-10).**
-
 # Datadog Trace Client
 
 [![Gem](https://img.shields.io/gem/v/ddtrace)](https://rubygems.org/gems/ddtrace/)
-[![CircleCI](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master.svg?style=svg&circle-token=b0bd5ef866ec7f7b018f48731bb495f2d1372cc1)](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master)
 [![codecov](https://codecov.io/gh/DataDog/dd-trace-rb/branch/master/graph/badge.svg)](https://app.codecov.io/gh/DataDog/dd-trace-rb/branch/master)
-[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)](https://s3.amazonaws.com/gems.datadoghq.com/trace/docs/index.html)
+[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)][api docs]
 
 ``ddtrace`` is Datadog’s tracing client for Ruby. It is used to trace requests as they flow across web servers,
-databases and microservices so that developers have great visiblity into bottlenecks and troublesome requests.
+databases and microservices so that developers have great visibility into bottlenecks and troublesome requests.
 
 ## Getting started
 
-For a basic product overview, check out our [setup documentation][setup docs].
+**If you're upgrading from a 0.x version, check out our [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md#from-0x-to-10).**
+
+For a product overview, installation, and configuration check out our [documentation][public docs].
 
-For installation, configuration, and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs].
+For the gem API, check out our [API documentation][api docs].
 
-For descriptions of terminology used in APM, take a look at the [official documentation][visualization docs].
+For descriptions of terminology used in APM, take a look at the [APM Terms and Concepts][APM glossary].
 
 For contributing, checkout the [contribution guidelines][contribution docs] and [development guide][development docs].
 
-[setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/
-[api docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/GettingStarted.md
-[gem docs]: https://s3.amazonaws.com/gems.datadoghq.com/trace/docs/index.html
-[visualization docs]: https://docs.datadoghq.com/tracing/visualization/
+[public docs]: https://docs.datadoghq.com/tracing/setup/ruby/
+[api docs]: https://datadog.github.io/dd-trace-rb/
+[APM glossary]: https://docs.datadoghq.com/tracing/glossary/
 [contribution docs]: https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md
 [development docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/DevelopmentGuide.md
 

data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c

@@ -29,10 +29,17 @@
   #define RTLD_DEEPBIND 0
 #endif
 
+// Used to mark function arguments that are deliberately left unused
+#ifdef __GNUC__
+  #define DDTRACE_UNUSED  __attribute__((unused))
+#else
+  #define DDTRACE_UNUSED
+#endif
+
 static VALUE ok_symbol = Qnil; // :ok in Ruby
 static VALUE error_symbol = Qnil; // :error in Ruby
 
-static VALUE _native_load(VALUE self, VALUE ruby_path, VALUE ruby_init_name);
+static VALUE _native_load(DDTRACE_UNUSED VALUE self, VALUE ruby_path, VALUE ruby_init_name);
 static bool failed_to_load(void *handle, VALUE *failure_details);
 static bool incompatible_library(void *handle, VALUE *failure_details);
 static bool failed_to_initialize(void *handle, char *init_name, VALUE *failure_details);
@@ -51,7 +58,7 @@ void DDTRACE_EXPORT Init_ddtrace_profiling_loader(void) {
   error_symbol = ID2SYM(rb_intern_const("error"));
 }
 
-static VALUE _native_load(VALUE self, VALUE ruby_path, VALUE ruby_init_name) {
+static VALUE _native_load(DDTRACE_UNUSED VALUE self, VALUE ruby_path, VALUE ruby_init_name) {
   Check_Type(ruby_path, T_STRING);
   Check_Type(ruby_init_name, T_STRING);
 

data/ext/ddtrace_profiling_loader/extconf.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 # rubocop:disable Style/StderrPuts
 # rubocop:disable Style/GlobalVars
 
@@ -24,7 +22,17 @@ def add_compiler_flag(flag)
   end
 end
 
-# Gets really noisy when we include the MJIT header, let's omit it
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true'
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another ddtrace dependency
+#   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
+add_compiler_flag '-std=gnu99'
+
+# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 add_compiler_flag '-Wno-unused-function'
 
 # Allow defining variables at any point in a function
@@ -34,6 +42,9 @@ add_compiler_flag '-Wno-declaration-after-statement'
 # cause a segfault later. Let's ensure that never happens.
 add_compiler_flag '-Werror-implicit-function-declaration'
 
+# Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+add_compiler_flag '-Wunused-parameter'
+
 # The native extension is not intended to expose any symbols/functions for other native libraries to use;
 # the sole exception being `Init_ddtrace_profiling_loader` which needs to be visible for Ruby to call it when
 # it `dlopen`s the library.
@@ -42,6 +53,13 @@ add_compiler_flag '-Werror-implicit-function-declaration'
 # For more details see https://gcc.gnu.org/wiki/Visibility
 add_compiler_flag '-fvisibility=hidden'
 
+# Avoid legacy C definitions
+add_compiler_flag '-Wold-style-definition'
+
+# Enable all other compiler warnings
+add_compiler_flag '-Wall'
+add_compiler_flag '-Wextra'
+
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
 # the wrong library is never loaded.

data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md

@@ -1,6 +1,7 @@
 # Profiling Native Extension Design
 
 The profiling native extension is used to:
+
 1. Implement features which are expensive (in terms of resources) or otherwise impossible to implement using Ruby code.
 2. Bridge between Ruby-specific profiling features and [`libdatadog`](https://github.com/DataDog/libdatadog), a Rust-based
 library with common profiling functionality.
@@ -15,13 +16,12 @@ the gem. Setting `DD_PROFILING_NO_EXTENSION` at installation time skips compilat
 
 (If you're a customer and needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.)
 
-In past releases, it was possible for the profiler to run without the native extension, but that's no longer the case,
-and disabling the extension will disable profiling.
+Disabling the profiler extension will disable profiling.
 
 ## Who is this page for?
 
-This documentation is intended to be used by dd-trace-rb developers. Please see the `docs/` folder for user-level
-documentation.
+**This documentation is intended to be used by dd-trace-rb developers. Please see the `docs/` folder for user-level
+documentation.**
 
 ## Must not block or break users that cannot use it
 
@@ -56,7 +56,7 @@ Non-exhaustive list of APIs that cause exceptions to be raised:
 
 * `Check_TypedStruct`, `Check_Type`, `ENFORCE_TYPE`
 * `rb_funcall`
-* `rb_thread_call_without_gvl`, `rb_thread_call_without_gvl2`
+* `rb_thread_call_without_gvl`
 * [Numeric conversion APIs, e.g. `NUM2LONG`, `NUM2INT`, etc.](https://silverhammermba.github.io/emberb/c/?#translation)
 * Our `char_slice_from_ruby_string` helper
 
@@ -67,28 +67,30 @@ internal types, structures and functions).
 
 Because these private header files are not included in regular Ruby installations, we have two different workarounds:
 
-1. for Ruby versions >= 2.6 we make use use the Ruby private MJIT header
-2. for Ruby versions < 2.6 (legacy Rubies) we make use of the `debase-ruby_core_source` gem
+1. for Ruby versions 2.6 to 3.2 we make use use the Ruby private MJIT header
+2. for Ruby versions < 2.6 and > 3.2 we make use of the `debase-ruby_core_source` gem
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
-There is currently no way for disabling usage of the private MJIT header for Ruby 2.6+.
+There is currently no way for disabling usage of the private MJIT header for Ruby 2.6 to 3.2.
 
 **Important Note**: Our medium/long-term plan is to stop relying on all private Ruby headers, and instead request and
 contribute upstream changes so that they become official public VM APIs.
 
 ### Approach 1: Using the Ruby private MJIT header
 
-Ruby versions >= 2.6 introduced a JIT compiler called MJIT. This compiler does not directly generate machine code;
+Ruby versions 2.6 to 3.2 shipped a JIT compiler called MJIT. This compiler does not directly generate machine code;
 instead it generates C code and uses the system C compiler to turn it into machine code.
 
-The generated C code `#include`s a private header -- which we reference as "the MJIT header" everywhere.
+The generated C code `#include`s a private header -- which we call "the MJIT header".
 The MJIT header gets shipped with all MJIT-enabled Rubies and includes the layout of many internal VM structures;
 and of course the intention is that it is only used by the Ruby MJIT compiler.
 
 This header is placed inside the `include/` directory in a Ruby installation, and is named for that specific Ruby
 version. e.g. `rb_mjit_min_header-2.7.4.h`.
 
+This header was removed in Ruby 3.3.
+
 ### Approach 2: Using the `debase-ruby_core_source` gem
 
 The [`debase-ruby_core_source`](https://github.com/ruby-debug/debase-ruby_core_source) contains almost no code;
@@ -101,8 +103,7 @@ Thus, even though a regular Ruby installation does not include these files, we c
 * **OS support**: Linux
 
 To enable CPU-time profiling, we use the `pthread_getcpuclockid(pthread_t thread, clockid_t *clockid)` C function to
-obtain a `clockid_t` that can then be used with the `Process.clock_gettime` method (or directly with the
-`clock_gettime()` C function).
+obtain a `clockid_t` that can then be used with the `clock_gettime` function.
 
 The challenge with using `pthread_getcpuclockid()` is that we need to get the `pthread_t` for a given Ruby `Thread`
 object. We previously did this with a weird combination of monkey patching and `pthread_self()` (effectively patching
@@ -118,3 +119,38 @@ Note that `pthread_getcpuclockid()` is not available on macOS (nor, obviously, o
 is currently Linux-specific. Thus, in the <clock_id_from_pthread.c> file we implement the feature for supported Ruby
 setups but if something is missing we instead compile in <clock_id_noop.c> that includes a no-op implementation of the
 feature.
+
+## Fork-safety
+
+It's common for Ruby applications to create child processes via the use of `fork`. For instance, this strategy is used
+by the puma webserver and the resque job processing tool.
+
+Thus, the profiler needs to be designed to take this into account. I'll call out two important parts of this design:
+
+1. Automatically propagate profiler to child processes. To make onboarding easier, we monkey patch the Ruby `fork` APIs
+so that the profiler is automatically restarted in child processes. This way, the user only needs to start profiling at
+the beginning of their application, and automatically forks are profiled as well.
+
+2. The profiler must ensure correctness and stability even if the application forks. There must be no impact on the
+application or incorrect data generated.
+
+### Fork-safety for libdatadog
+
+Since libdatadog is built in native code (Rust), special care needs to be take to consider how we're using it and how
+it can be affected by the use of `fork`.
+
+* Profile-related APIs: `Profile_new` and `Profile_add` and `Profile_free` are only called with the Ruby Global VM Lock
+being held. Thus, if Ruby APIs are being used for fork, this prevents any concurrency between profile mutation and
+forking, because if we’re holding the lock, then no other thread can call into the fork APIs.
+(Calling libc `fork()` directly from a native extension is possible but would break the VM as well, since it does need
+to do some of its own work when forking happens, so we’ll ignore that one)
+
+* Exporter-related APIs: Explicitly to make sure we had no issues with forking, we create a new `CancellationToken_new`
+and `ProfileExporterV3_new` for every report. We do release the Global VM Lock during exporting, so it's possible for
+forking and exporting to be concurrent.
+
+  Both the CancellationToken and ProfileExporter are only referenced on the stack of the thread doing the exporting, so
+  they will not be reused in the child process after the fork. In the worst case, if a report is concurrent with a fork,
+  then it's possible a small amount of memory will not be cleaned up in the child process.
+
+  Because there is no leftover undefined state, we guarantee correctness for the exporter APIs.

data/ext/ddtrace_profiling_native_extension/clock_id.h

@@ -17,8 +17,6 @@ typedef struct thread_cpu_time {
 
 void self_test_clock_id(void);
 
-// TODO: Remove this after the OldStack profiler gets removed
-VALUE clock_id_for(VALUE self, VALUE thread);
-
+// Safety: This function is assumed never to raise exceptions by callers
 thread_cpu_time_id thread_cpu_time_id_for(VALUE thread);
 thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id);

data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c

@@ -7,11 +7,12 @@
 #include <pthread.h>
 #include <time.h>
 #include <errno.h>
-
 #include <ruby.h>
+
 #include "helpers.h"
 #include "private_vm_api_access.h"
 #include "clock_id.h"
+#include "time_helpers.h"
 
 // Validate that our home-cooked pthread_id_for() matches pthread_self() for the current thread
 void self_test_clock_id(void) {
@@ -21,30 +22,12 @@ void self_test_clock_id(void) {
   if (expected_pthread_id != actual_pthread_id) rb_raise(rb_eRuntimeError, "pthread_id_for() self-test failed");
 }
 
-// TODO: Remove this after the OldStack profiler gets removed
-VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, VALUE thread) {
+// Safety: This function is assumed never to raise exceptions by callers
+thread_cpu_time_id thread_cpu_time_id_for(VALUE thread) {
   rb_nativethread_id_t thread_id = pthread_id_for(thread);
 
-  clockid_t clock_id;
-  int error = pthread_getcpuclockid(thread_id, &clock_id);
-
-  if (error == 0) {
-    return CLOCKID2NUM(clock_id);
-  } else {
-    switch(error) {
-      // The more specific error messages are based on the pthread_getcpuclockid(3) man page
-      case ENOENT:
-        rb_exc_raise(rb_syserr_new(error, "Failed to get clock_id for given thread: Per-thread CPU time clocks are not supported by the system."));
-      case ESRCH:
-        rb_exc_raise(rb_syserr_new(error, "Failed to get clock_id for given thread: No thread could be found."));
-      default:
-        rb_exc_raise(rb_syserr_new(error, "Failed to get clock_id for given thread"));
-    }
-  }
-}
+  if (thread_id == 0) return (thread_cpu_time_id) {.valid = false};
 
-thread_cpu_time_id thread_cpu_time_id_for(VALUE thread) {
-  rb_nativethread_id_t thread_id = pthread_id_for(thread);
   clockid_t clock_id;
 
   int error = pthread_getcpuclockid(thread_id, &clock_id);
@@ -67,7 +50,7 @@ thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id) {
   // TODO: Include the error code in some way in the output?
   if (clock_gettime(time_id.clock_id, &current_cpu) != 0) return error;
 
-  return (thread_cpu_time) {.valid = true, .result_ns = current_cpu.tv_nsec + (current_cpu.tv_sec * 1000 * 1000 * 1000)};
+  return (thread_cpu_time) {.valid = true, .result_ns = current_cpu.tv_nsec + SECONDS_AS_NS(current_cpu.tv_sec)};
 }
 
 #endif

data/ext/ddtrace_profiling_native_extension/clock_id_noop.c

@@ -10,7 +10,6 @@
 #include "helpers.h"
 
 void self_test_clock_id(void) { } // Nothing to check
-VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, DDTRACE_UNUSED VALUE _thread) { return Qnil; } // Nothing to return
 
 thread_cpu_time_id thread_cpu_time_id_for(DDTRACE_UNUSED VALUE _thread) {
   return (thread_cpu_time_id) {.valid = false};