ddtrace 1.1.0 → 1.12.1

data/LICENSE-3rdparty.csv

@@ -1,4 +1,8 @@
 Component,Origin,License,Copyright
 lib/datadog/core/vendor/multipart-post,https://github.com/socketry/multipart-post,MIT,"Copyright (c) 2007-2013 Nick Sieger."
 lib/datadog/tracing/contrib/active_record/vendor,https://github.com/rails/rails/,MIT,"Copyright (c) 2005-2018 David Heinemeier Hansson"
+lib/datadog/tracing/contrib/utils/quantization/http.rb,https://github.com/ruby/uri,BSD-2-Clause,"Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
 ext/ddtrace_profiling_native_extension/private_vm_api_access,https://github.com/ruby/ruby,BSD-2-Clause,"Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
+msgpack,https://rubygems.org/gems/msgpack,Apache-2.0,"Copyright (c) 2008-2015 Sadayuki Furuhashi"
+debase-ruby_core_source,https://rubygems.org/gems/debase-ruby_core_source,MIT for gem and BSD-2-Clause for Ruby sources,"Copyright (c) 2012 Gabriel Horner. Files from Ruby sources are Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved."
+lib/datadog/core/vendor/ipaddr,https://github.com/ruby/ipaddr/blob/master/lib/ipaddr.rb,BSD 2-Clause "Simplified" License,"Copyright (c) 2002 Hajimu UMEMOTO <ume@mahoroba.org> Copyright (c) 2007-2017 Akinori MUSHA <knu@iDaemons.org>"

data/README.md

@@ -1,28 +1,31 @@
-***Version 1.0.0 has recently been released. Check out our [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md#from-0x-to-10) for more details.***
-
 # Datadog Trace Client
 
 [![Gem](https://img.shields.io/gem/v/ddtrace)](https://rubygems.org/gems/ddtrace/)
-[![CircleCI](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master.svg?style=svg&circle-token=b0bd5ef866ec7f7b018f48731bb495f2d1372cc1)](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master)
 [![codecov](https://codecov.io/gh/DataDog/dd-trace-rb/branch/master/graph/badge.svg)](https://app.codecov.io/gh/DataDog/dd-trace-rb/branch/master)
-[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)](https://www.rubydoc.info/gems/ddtrace/)
+[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)][api docs]
 
 ``ddtrace`` is Datadog’s tracing client for Ruby. It is used to trace requests as they flow across web servers,
-databases and microservices so that developers have great visiblity into bottlenecks and troublesome requests.
+databases and microservices so that developers have great visibility into bottlenecks and troublesome requests.
 
 ## Getting started
 
-For a basic product overview, check out our [setup documentation][setup docs].
+**If you're upgrading from a 0.x version, check out our [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md#from-0x-to-10).**
+
+For a product overview, installation, and configuration check out our [documentation][public docs].
 
-For installation, configuration, and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs].
+For the gem API, check out our [API documentation][api docs].
 
-For descriptions of terminology used in APM, take a look at the [official documentation][visualization docs].
+For descriptions of terminology used in APM, take a look at the [APM Terms and Concepts][APM glossary].
 
 For contributing, checkout the [contribution guidelines][contribution docs] and [development guide][development docs].
 
-[setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/
-[api docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/GettingStarted.md
-[gem docs]: http://gems.datadoghq.com/trace/docs/
-[visualization docs]: https://docs.datadoghq.com/tracing/visualization/
+[public docs]: https://docs.datadoghq.com/tracing/setup/ruby/
+[api docs]: https://datadog.github.io/dd-trace-rb/
+[APM glossary]: https://docs.datadoghq.com/tracing/glossary/
 [contribution docs]: https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md
 [development docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/DevelopmentGuide.md
+
+## Special thanks
+
+* [Mike Fiedler](https://github.com/miketheman) for working on a number of Datadog Ruby projects, as well as graciously
+  gifting control of the `datadog` gem

data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c

@@ -20,7 +20,7 @@
 // This idea was shamelessly stolen from @lloeki's work in https://github.com/rubyjs/mini_racer/pull/179, big thanks!
 //
 // Extra note: Currently (May 2022), that we know of, the profiling native extension only exposes one potentially
-// problematic symbol: `rust_eh_personality` (coming from libddprof/libdatadog).
+// problematic symbol: `rust_eh_personality` (coming from libdatadog).
 // Future versions of Rust have been patched not to expose this
 // (see https://github.com/rust-lang/rust/pull/95604#issuecomment-1108563434) so we may want to revisit the need
 // for this loader in the future, and perhaps delete it if we no longer require its services :)
@@ -29,10 +29,17 @@
   #define RTLD_DEEPBIND 0
 #endif
 
+// Used to mark function arguments that are deliberately left unused
+#ifdef __GNUC__
+  #define DDTRACE_UNUSED  __attribute__((unused))
+#else
+  #define DDTRACE_UNUSED
+#endif
+
 static VALUE ok_symbol = Qnil; // :ok in Ruby
 static VALUE error_symbol = Qnil; // :error in Ruby
 
-static VALUE _native_load(VALUE self, VALUE ruby_path, VALUE ruby_init_name);
+static VALUE _native_load(DDTRACE_UNUSED VALUE self, VALUE ruby_path, VALUE ruby_init_name);
 static bool failed_to_load(void *handle, VALUE *failure_details);
 static bool incompatible_library(void *handle, VALUE *failure_details);
 static bool failed_to_initialize(void *handle, char *init_name, VALUE *failure_details);
@@ -41,7 +48,7 @@ static void unload_failed_library(void *handle);
 
 #define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
 
-void DDTRACE_EXPORT Init_ddtrace_profiling_loader() {
+void DDTRACE_EXPORT Init_ddtrace_profiling_loader(void) {
   VALUE datadog_module = rb_define_module("Datadog");
   VALUE profiling_module = rb_define_module_under(datadog_module, "Profiling");
   VALUE loader_module = rb_define_module_under(profiling_module, "Loader");
@@ -51,7 +58,7 @@ void DDTRACE_EXPORT Init_ddtrace_profiling_loader() {
   error_symbol = ID2SYM(rb_intern_const("error"));
 }
 
-static VALUE _native_load(VALUE self, VALUE ruby_path, VALUE ruby_init_name) {
+static VALUE _native_load(DDTRACE_UNUSED VALUE self, VALUE ruby_path, VALUE ruby_init_name) {
   Check_Type(ruby_path, T_STRING);
   Check_Type(ruby_init_name, T_STRING);
 
@@ -85,7 +92,16 @@ static bool failed_to_load(void *handle, VALUE *failure_details) {
 static bool incompatible_library(void *handle, VALUE *failure_details) {
   // The library being loaded may be linked to a different libruby than the current executing Ruby.
   // We check if this is the case by checking if a well-known symbol resolves to a common address.
-  if (dlsym(handle, "ruby_xmalloc") != &ruby_xmalloc) {
+
+  void *xmalloc_from_library = dlsym(handle, "ruby_xmalloc");
+
+  if (xmalloc_from_library == NULL) {
+    // This happens when ruby is built without a `libruby.so` by using `--disable-shared` at compilation time.
+    // In this situation, no conflict between libruby version is possible.
+    return false;
+  }
+
+  if (xmalloc_from_library != &ruby_xmalloc) {
     *failure_details = rb_str_new_cstr("library was compiled and linked to a different Ruby version");
     unload_failed_library(handle);
     return true;

data/ext/ddtrace_profiling_loader/extconf.rb

@@ -1,4 +1,3 @@
-# typed: ignore
 # rubocop:disable Style/StderrPuts
 # rubocop:disable Style/GlobalVars
 
@@ -23,7 +22,17 @@ def add_compiler_flag(flag)
   end
 end
 
-# Gets really noisy when we include the MJIT header, let's omit it
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true'
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another ddtrace dependency
+#   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
+add_compiler_flag '-std=gnu99'
+
+# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 add_compiler_flag '-Wno-unused-function'
 
 # Allow defining variables at any point in a function
@@ -33,6 +42,9 @@ add_compiler_flag '-Wno-declaration-after-statement'
 # cause a segfault later. Let's ensure that never happens.
 add_compiler_flag '-Werror-implicit-function-declaration'
 
+# Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+add_compiler_flag '-Wunused-parameter'
+
 # The native extension is not intended to expose any symbols/functions for other native libraries to use;
 # the sole exception being `Init_ddtrace_profiling_loader` which needs to be visible for Ruby to call it when
 # it `dlopen`s the library.
@@ -41,6 +53,13 @@ add_compiler_flag '-Werror-implicit-function-declaration'
 # For more details see https://gcc.gnu.org/wiki/Visibility
 add_compiler_flag '-fvisibility=hidden'
 
+# Avoid legacy C definitions
+add_compiler_flag '-Wold-style-definition'
+
+# Enable all other compiler warnings
+add_compiler_flag '-Wall'
+add_compiler_flag '-Wextra'
+
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
 # the wrong library is never loaded.

data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md

@@ -1,8 +1,9 @@
 # Profiling Native Extension Design
 
 The profiling native extension is used to:
+
 1. Implement features which are expensive (in terms of resources) or otherwise impossible to implement using Ruby code.
-2. Bridge between Ruby-specific profiling features and [`libddprof`](https://github.com/DataDog/libddprof), a Rust-based
+2. Bridge between Ruby-specific profiling features and [`libdatadog`](https://github.com/DataDog/libdatadog), a Rust-based
 library with common profiling functionality.
 
 Due to (1), this extension is quite coupled with MRI Ruby ("C Ruby") internals, and is not intended to support other rubies such as
@@ -10,13 +11,18 @@ JRuby or TruffleRuby. When below we say "Ruby", read it as "MRI Ruby".
 
 ## Disabling
 
-The profiling native extension can be disabled by setting `DD_PROFILING_NO_EXTENSION=true` when installing or running
+The profiling native extension can be disabled by setting `DD_PROFILING_NO_EXTENSION=true` when installing
 the gem. Setting `DD_PROFILING_NO_EXTENSION` at installation time skips compilation of the extension entirely.
 
 (If you're a customer and needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.)
 
-Currently the profiler can still "limp along" when the native extension is disabled, but the plan is to require it
-in future releases -- e.g. disabling the extension will disable profiling entirely.
+In past releases, it was possible for the profiler to run without the native extension, but that's no longer the case,
+and disabling the extension will disable profiling.
+
+## Who is this page for?
+
+**This documentation is intended to be used by dd-trace-rb developers. Please see the `docs/` folder for user-level
+documentation.**
 
 ## Must not block or break users that cannot use it
 
@@ -44,13 +50,15 @@ We avoid issues using a combination of:
 * Avoiding calling Ruby VM APIs after doing dynamic allocations
 * Wrapping dynamic allocations into Ruby GC-managed objects (using `TypedData_Wrap_Struct`), so that Ruby will manage
   their lifetime and call `free` when the GC-managed object is no longer being referenced
+* Using [`rb_protect` and similar APIs](https://silverhammermba.github.io/emberb/c/?#rescue) to run cleanup code on
+  exception cases
 
 Non-exhaustive list of APIs that cause exceptions to be raised:
 
-* `Check_TypedStruct`, `Check_Type`
+* `Check_TypedStruct`, `Check_Type`, `ENFORCE_TYPE`
 * `rb_funcall`
-* `rb_thread_call_without_gvl`, `rb_thread_call_without_gvl2`
-* [Numeric conversion APIs, e.g. `NUM2LONG`, `NUM2INT`, etc.](https://silverhammermba.github.io/emberb/c/?utm_source=pocket_mylist#translation)
+* `rb_thread_call_without_gvl`
+* [Numeric conversion APIs, e.g. `NUM2LONG`, `NUM2INT`, etc.](https://silverhammermba.github.io/emberb/c/?#translation)
 * Our `char_slice_from_ruby_string` helper
 
 ## Usage of private VM headers
@@ -60,17 +68,19 @@ internal types, structures and functions).
 
 Because these private header files are not included in regular Ruby installations, we have two different workarounds:
 
-1. for Ruby versions >= 2.6 we make use use the Ruby private MJIT header
-2. for Ruby versions < 2.6 (legacy Rubies) we make use of the `debase-ruby_core_source` gem
+1. for Ruby versions 2.6 to 3.2 we make use use the Ruby private MJIT header
+2. for Ruby versions < 2.6 and > 3.2 we make use of the `debase-ruby_core_source` gem
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
+There is currently no way for disabling usage of the private MJIT header for Ruby 2.6 to 3.2.
+
 **Important Note**: Our medium/long-term plan is to stop relying on all private Ruby headers, and instead request and
 contribute upstream changes so that they become official public VM APIs.
 
 ### Approach 1: Using the Ruby private MJIT header
 
-Ruby versions >= 2.6 introduced a JIT compiler called MJIT. This compiler does not directly generate machine code;
+Ruby versions 2.6 to 3.2 shipped a JIT compiler called MJIT. This compiler does not directly generate machine code;
 instead it generates C code and uses the system C compiler to turn it into machine code.
 
 The generated C code `#include`s a private header -- which we reference as "the MJIT header" everywhere.
@@ -80,6 +90,8 @@ and of course the intention is that it is only used by the Ruby MJIT compiler.
 This header is placed inside the `include/` directory in a Ruby installation, and is named for that specific Ruby
 version. e.g. `rb_mjit_min_header-2.7.4.h`.
 
+This header was removed in Ruby 3.3.
+
 ### Approach 2: Using the `debase-ruby_core_source` gem
 
 The [`debase-ruby_core_source`](https://github.com/ruby-debug/debase-ruby_core_source) contains almost no code;
@@ -90,7 +102,6 @@ Thus, even though a regular Ruby installation does not include these files, we c
 ## Feature: Getting thread CPU-time clock_ids
 
 * **OS support**: Linux
-* **Ruby support**: 2.6+
 
 To enable CPU-time profiling, we use the `pthread_getcpuclockid(pthread_t thread, clockid_t *clockid)` C function to
 obtain a `clockid_t` that can then be used with the `Process.clock_gettime` method (or directly with the
@@ -110,3 +121,38 @@ Note that `pthread_getcpuclockid()` is not available on macOS (nor, obviously, o
 is currently Linux-specific. Thus, in the <clock_id_from_pthread.c> file we implement the feature for supported Ruby
 setups but if something is missing we instead compile in <clock_id_noop.c> that includes a no-op implementation of the
 feature.
+
+## Fork-safety
+
+It's common for Ruby applications to create child processes via the use of `fork`. For instance, this strategy is used
+by the puma webserver and the resque job processing tool.
+
+Thus, the profiler needs to be designed to take this into account. I'll call out two important parts of this design:
+
+1. Automatically propagate profiler to child processes. To make onboarding easier, we monkey patch the Ruby `fork` APIs
+so that the profiler is automatically restarted in child processes. This way, the user only needs to start profiling at
+the beginning of their application, and automatically forks are profiled as well.
+
+2. The profiler must ensure correctness and stability even if the application forks. There must be no impact on the
+application or incorrect data generated.
+
+### Fork-safety for libdatadog
+
+Since libdatadog is built in native code (Rust), special care needs to be take to consider how we're using it and how
+it can be affected by the use of `fork`.
+
+* Profile-related APIs: `Profile_new` and `Profile_add` and `Profile_free` are only called with the Ruby Global VM Lock
+being held. Thus, if Ruby APIs are being used for fork, this prevents any concurrency between profile mutation and
+forking, because if we’re holding the lock, then no other thread can call into the fork APIs.
+(Calling libc `fork()` directly from a native extension is possible but would break the VM as well, since it does need
+to do some of its own work when forking happens, so we’ll ignore that one)
+
+* Exporter-related APIs: Explicitly to make sure we had no issues with forking, we create a new `CancellationToken_new`
+and `ProfileExporterV3_new` for every report. We do release the Global VM Lock during exporting, so it's possible for
+forking and exporting to be concurrent.
+
+  Both the CancellationToken and ProfileExporter are only referenced on the stack of the thread doing the exporting, so
+  they will not be reused in the child process after the fork. In the worst case, if a report is concurrent with a fork,
+  then it's possible a small amount of memory will not be cleaned up in the child process.
+
+  Because there is no leftover undefined state, we guarantee correctness for the exporter APIs.

data/ext/ddtrace_profiling_native_extension/clock_id.h

@@ -1,4 +1,25 @@
 #pragma once
 
-void self_test_clock_id();
+#include <stdbool.h>
+#include <time.h>
+
+// Contains the operating-system specific identifier needed to fetch CPU-time, and a flag to indicate if we failed to fetch it
+typedef struct thread_cpu_time_id {
+  bool valid;
+  clockid_t clock_id;
+} thread_cpu_time_id;
+
+// Contains the current cpu time, and a flag to indicate if we failed to fetch it
+typedef struct thread_cpu_time {
+  bool valid;
+  long result_ns;
+} thread_cpu_time;
+
+void self_test_clock_id(void);
+
+// TODO: Remove this after the OldStack profiler gets removed
 VALUE clock_id_for(VALUE self, VALUE thread);
+
+// Safety: This function is assumed never to raise exceptions by callers
+thread_cpu_time_id thread_cpu_time_id_for(VALUE thread);
+thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id);

data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c

@@ -7,20 +7,23 @@
 #include <pthread.h>
 #include <time.h>
 #include <errno.h>
-
 #include <ruby.h>
+
+#include "helpers.h"
 #include "private_vm_api_access.h"
 #include "clock_id.h"
+#include "time_helpers.h"
 
 // Validate that our home-cooked pthread_id_for() matches pthread_self() for the current thread
-void self_test_clock_id() {
+void self_test_clock_id(void) {
   rb_nativethread_id_t expected_pthread_id = pthread_self();
   rb_nativethread_id_t actual_pthread_id = pthread_id_for(rb_thread_current());
 
   if (expected_pthread_id != actual_pthread_id) rb_raise(rb_eRuntimeError, "pthread_id_for() self-test failed");
 }
 
-VALUE clock_id_for(VALUE self, VALUE thread) {
+// TODO: Remove this after the OldStack profiler gets removed
+VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, VALUE thread) {
   rb_nativethread_id_t thread_id = pthread_id_for(thread);
 
   clockid_t clock_id;
@@ -41,4 +44,32 @@ VALUE clock_id_for(VALUE self, VALUE thread) {
   }
 }
 
+// Safety: This function is assumed never to raise exceptions by callers
+thread_cpu_time_id thread_cpu_time_id_for(VALUE thread) {
+  rb_nativethread_id_t thread_id = pthread_id_for(thread);
+  clockid_t clock_id;
+
+  int error = pthread_getcpuclockid(thread_id, &clock_id);
+
+  if (error == 0) {
+    return (thread_cpu_time_id) {.valid = true, .clock_id = clock_id};
+  } else {
+    // TODO: Include the error code in some way in the output?
+    return (thread_cpu_time_id) {.valid = false};
+  }
+}
+
+thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id) {
+  thread_cpu_time error = (thread_cpu_time) {.valid = false};
+
+  if (!time_id.valid) { return error; }
+
+  struct timespec current_cpu;
+
+  // TODO: Include the error code in some way in the output?
+  if (clock_gettime(time_id.clock_id, &current_cpu) != 0) return error;
+
+  return (thread_cpu_time) {.valid = true, .result_ns = current_cpu.tv_nsec + SECONDS_AS_NS(current_cpu.tv_sec)};
+}
+
 #endif

data/ext/ddtrace_profiling_native_extension/clock_id_noop.c

@@ -7,8 +7,17 @@
 #include <ruby.h>
 
 #include "clock_id.h"
+#include "helpers.h"
 
-void self_test_clock_id() { } // Nothing to check
-VALUE clock_id_for(VALUE self, VALUE thread) { return Qnil; } // Nothing to return
+void self_test_clock_id(void) { } // Nothing to check
+VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, DDTRACE_UNUSED VALUE _thread) { return Qnil; } // Nothing to return
+
+thread_cpu_time_id thread_cpu_time_id_for(DDTRACE_UNUSED VALUE _thread) {
+  return (thread_cpu_time_id) {.valid = false};
+}
+
+thread_cpu_time thread_cpu_time_for(DDTRACE_UNUSED thread_cpu_time_id _time_id) {
+  return (thread_cpu_time) {.valid = false};
+}
 
 #endif