ddtrace 1.1.0 → 1.9.0

data/ext/ddtrace_profiling_loader/extconf.rb

@@ -1,4 +1,5 @@
 # typed: ignore
+
 # rubocop:disable Style/StderrPuts
 # rubocop:disable Style/GlobalVars
 
@@ -23,7 +24,17 @@ def add_compiler_flag(flag)
   end
 end
 
-# Gets really noisy when we include the MJIT header, let's omit it
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+add_compiler_flag '-Werror' if ENV['DDTRACE_CI'] == 'true'
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another ddtrace dependency
+#   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
+add_compiler_flag '-std=gnu99'
+
+# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
 add_compiler_flag '-Wno-unused-function'
 
 # Allow defining variables at any point in a function
@@ -33,6 +44,9 @@ add_compiler_flag '-Wno-declaration-after-statement'
 # cause a segfault later. Let's ensure that never happens.
 add_compiler_flag '-Werror-implicit-function-declaration'
 
+# Warn on unused parameters to functions. Use `DDTRACE_UNUSED` to mark things as known-to-not-be-used.
+add_compiler_flag '-Wunused-parameter'
+
 # The native extension is not intended to expose any symbols/functions for other native libraries to use;
 # the sole exception being `Init_ddtrace_profiling_loader` which needs to be visible for Ruby to call it when
 # it `dlopen`s the library.
@@ -41,6 +55,13 @@ add_compiler_flag '-Werror-implicit-function-declaration'
 # For more details see https://gcc.gnu.org/wiki/Visibility
 add_compiler_flag '-fvisibility=hidden'
 
+# Avoid legacy C definitions
+add_compiler_flag '-Wold-style-definition'
+
+# Enable all other compiler warnings
+add_compiler_flag '-Wall'
+add_compiler_flag '-Wextra'
+
 # Tag the native extension library with the Ruby version and Ruby platform.
 # This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
 # the wrong library is never loaded.

data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md

@@ -1,8 +1,9 @@
 # Profiling Native Extension Design
 
 The profiling native extension is used to:
+
 1. Implement features which are expensive (in terms of resources) or otherwise impossible to implement using Ruby code.
-2. Bridge between Ruby-specific profiling features and [`libddprof`](https://github.com/DataDog/libddprof), a Rust-based
+2. Bridge between Ruby-specific profiling features and [`libdatadog`](https://github.com/DataDog/libdatadog), a Rust-based
 library with common profiling functionality.
 
 Due to (1), this extension is quite coupled with MRI Ruby ("C Ruby") internals, and is not intended to support other rubies such as
@@ -10,13 +11,18 @@ JRuby or TruffleRuby. When below we say "Ruby", read it as "MRI Ruby".
 
 ## Disabling
 
-The profiling native extension can be disabled by setting `DD_PROFILING_NO_EXTENSION=true` when installing or running
+The profiling native extension can be disabled by setting `DD_PROFILING_NO_EXTENSION=true` when installing
 the gem. Setting `DD_PROFILING_NO_EXTENSION` at installation time skips compilation of the extension entirely.
 
 (If you're a customer and needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.)
 
-Currently the profiler can still "limp along" when the native extension is disabled, but the plan is to require it
-in future releases -- e.g. disabling the extension will disable profiling entirely.
+In past releases, it was possible for the profiler to run without the native extension, but that's no longer the case,
+and disabling the extension will disable profiling.
+
+## Who is this page for?
+
+**This documentation is intended to be used by dd-trace-rb developers. Please see the `docs/` folder for user-level
+documentation.**
 
 ## Must not block or break users that cannot use it
 
@@ -44,13 +50,15 @@ We avoid issues using a combination of:
 * Avoiding calling Ruby VM APIs after doing dynamic allocations
 * Wrapping dynamic allocations into Ruby GC-managed objects (using `TypedData_Wrap_Struct`), so that Ruby will manage
   their lifetime and call `free` when the GC-managed object is no longer being referenced
+* Using [`rb_protect` and similar APIs](https://silverhammermba.github.io/emberb/c/?#rescue) to run cleanup code on
+  exception cases
 
 Non-exhaustive list of APIs that cause exceptions to be raised:
 
-* `Check_TypedStruct`, `Check_Type`
+* `Check_TypedStruct`, `Check_Type`, `ENFORCE_TYPE`
 * `rb_funcall`
-* `rb_thread_call_without_gvl`, `rb_thread_call_without_gvl2`
-* [Numeric conversion APIs, e.g. `NUM2LONG`, `NUM2INT`, etc.](https://silverhammermba.github.io/emberb/c/?utm_source=pocket_mylist#translation)
+* `rb_thread_call_without_gvl`
+* [Numeric conversion APIs, e.g. `NUM2LONG`, `NUM2INT`, etc.](https://silverhammermba.github.io/emberb/c/?#translation)
 * Our `char_slice_from_ruby_string` helper
 
 ## Usage of private VM headers
@@ -65,6 +73,8 @@ Because these private header files are not included in regular Ruby installation
 
 Functions which make use of these headers are defined in the <private_vm_api_acccess.c> file.
 
+There is currently no way for disabling usage of the private MJIT header for Ruby 2.6+.
+
 **Important Note**: Our medium/long-term plan is to stop relying on all private Ruby headers, and instead request and
 contribute upstream changes so that they become official public VM APIs.
 
@@ -90,7 +100,6 @@ Thus, even though a regular Ruby installation does not include these files, we c
 ## Feature: Getting thread CPU-time clock_ids
 
 * **OS support**: Linux
-* **Ruby support**: 2.6+
 
 To enable CPU-time profiling, we use the `pthread_getcpuclockid(pthread_t thread, clockid_t *clockid)` C function to
 obtain a `clockid_t` that can then be used with the `Process.clock_gettime` method (or directly with the
@@ -110,3 +119,38 @@ Note that `pthread_getcpuclockid()` is not available on macOS (nor, obviously, o
 is currently Linux-specific. Thus, in the <clock_id_from_pthread.c> file we implement the feature for supported Ruby
 setups but if something is missing we instead compile in <clock_id_noop.c> that includes a no-op implementation of the
 feature.
+
+## Fork-safety
+
+It's common for Ruby applications to create child processes via the use of `fork`. For instance, this strategy is used
+by the puma webserver and the resque job processing tool.
+
+Thus, the profiler needs to be designed to take this into account. I'll call out two important parts of this design:
+
+1. Automatically propagate profiler to child processes. To make onboarding easier, we monkey patch the Ruby `fork` APIs
+so that the profiler is automatically restarted in child processes. This way, the user only needs to start profiling at
+the beginning of their application, and automatically forks are profiled as well.
+
+2. The profiler must ensure correctness and stability even if the application forks. There must be no impact on the
+application or incorrect data generated.
+
+### Fork-safety for libdatadog
+
+Since libdatadog is built in native code (Rust), special care needs to be take to consider how we're using it and how
+it can be affected by the use of `fork`.
+
+* Profile-related APIs: `Profile_new` and `Profile_add` and `Profile_free` are only called with the Ruby Global VM Lock
+being held. Thus, if Ruby APIs are being used for fork, this prevents any concurrency between profile mutation and
+forking, because if we’re holding the lock, then no other thread can call into the fork APIs.
+(Calling libc `fork()` directly from a native extension is possible but would break the VM as well, since it does need
+to do some of its own work when forking happens, so we’ll ignore that one)
+
+* Exporter-related APIs: Explicitly to make sure we had no issues with forking, we create a new `CancellationToken_new`
+and `ProfileExporterV3_new` for every report. We do release the Global VM Lock during exporting, so it's possible for
+forking and exporting to be concurrent.
+
+  Both the CancellationToken and ProfileExporter are only referenced on the stack of the thread doing the exporting, so
+  they will not be reused in the child process after the fork. In the worst case, if a report is concurrent with a fork,
+  then it's possible a small amount of memory will not be cleaned up in the child process.
+
+  Because there is no leftover undefined state, we guarantee correctness for the exporter APIs.

data/ext/ddtrace_profiling_native_extension/clock_id.h

@@ -1,4 +1,25 @@
 #pragma once
 
-void self_test_clock_id();
+#include <stdbool.h>
+#include <time.h>
+
+// Contains the operating-system specific identifier needed to fetch CPU-time, and a flag to indicate if we failed to fetch it
+typedef struct thread_cpu_time_id {
+  bool valid;
+  clockid_t clock_id;
+} thread_cpu_time_id;
+
+// Contains the current cpu time, and a flag to indicate if we failed to fetch it
+typedef struct thread_cpu_time {
+  bool valid;
+  long result_ns;
+} thread_cpu_time;
+
+void self_test_clock_id(void);
+
+// TODO: Remove this after the OldStack profiler gets removed
 VALUE clock_id_for(VALUE self, VALUE thread);
+
+// Safety: This function is assumed never to raise exceptions by callers
+thread_cpu_time_id thread_cpu_time_id_for(VALUE thread);
+thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id);

data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c

@@ -7,20 +7,23 @@
 #include <pthread.h>
 #include <time.h>
 #include <errno.h>
-
 #include <ruby.h>
+
+#include "helpers.h"
 #include "private_vm_api_access.h"
 #include "clock_id.h"
+#include "time_helpers.h"
 
 // Validate that our home-cooked pthread_id_for() matches pthread_self() for the current thread
-void self_test_clock_id() {
+void self_test_clock_id(void) {
   rb_nativethread_id_t expected_pthread_id = pthread_self();
   rb_nativethread_id_t actual_pthread_id = pthread_id_for(rb_thread_current());
 
   if (expected_pthread_id != actual_pthread_id) rb_raise(rb_eRuntimeError, "pthread_id_for() self-test failed");
 }
 
-VALUE clock_id_for(VALUE self, VALUE thread) {
+// TODO: Remove this after the OldStack profiler gets removed
+VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, VALUE thread) {
   rb_nativethread_id_t thread_id = pthread_id_for(thread);
 
   clockid_t clock_id;
@@ -41,4 +44,32 @@ VALUE clock_id_for(VALUE self, VALUE thread) {
   }
 }
 
+// Safety: This function is assumed never to raise exceptions by callers
+thread_cpu_time_id thread_cpu_time_id_for(VALUE thread) {
+  rb_nativethread_id_t thread_id = pthread_id_for(thread);
+  clockid_t clock_id;
+
+  int error = pthread_getcpuclockid(thread_id, &clock_id);
+
+  if (error == 0) {
+    return (thread_cpu_time_id) {.valid = true, .clock_id = clock_id};
+  } else {
+    // TODO: Include the error code in some way in the output?
+    return (thread_cpu_time_id) {.valid = false};
+  }
+}
+
+thread_cpu_time thread_cpu_time_for(thread_cpu_time_id time_id) {
+  thread_cpu_time error = (thread_cpu_time) {.valid = false};
+
+  if (!time_id.valid) { return error; }
+
+  struct timespec current_cpu;
+
+  // TODO: Include the error code in some way in the output?
+  if (clock_gettime(time_id.clock_id, &current_cpu) != 0) return error;
+
+  return (thread_cpu_time) {.valid = true, .result_ns = current_cpu.tv_nsec + SECONDS_AS_NS(current_cpu.tv_sec)};
+}
+
 #endif

data/ext/ddtrace_profiling_native_extension/clock_id_noop.c

@@ -7,8 +7,17 @@
 #include <ruby.h>
 
 #include "clock_id.h"
+#include "helpers.h"
 
-void self_test_clock_id() { } // Nothing to check
-VALUE clock_id_for(VALUE self, VALUE thread) { return Qnil; } // Nothing to return
+void self_test_clock_id(void) { } // Nothing to check
+VALUE clock_id_for(DDTRACE_UNUSED VALUE _self, DDTRACE_UNUSED VALUE _thread) { return Qnil; } // Nothing to return
+
+thread_cpu_time_id thread_cpu_time_id_for(DDTRACE_UNUSED VALUE _thread) {
+  return (thread_cpu_time_id) {.valid = false};
+}
+
+thread_cpu_time thread_cpu_time_for(DDTRACE_UNUSED thread_cpu_time_id _time_id) {
+  return (thread_cpu_time) {.valid = false};
+}
 
 #endif