gvl-tracing 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47df73ea238560430f56d7b1ee2f0484e9e57b8f0784e065209cd5cfbe7b1084
-  data.tar.gz: 1517522202fd357b31db4a087f769991ccf9f0a0c56b38191f64976827b3e752
+  metadata.gz: af27ef999bd0dba6070d9faf4e4465f77ae7dc440d77d99999be6ac0c1be07ec
+  data.tar.gz: 2a0978b7135b22a9827e7f040f04ba7e261bd3ebb9f576218e19473d84560053
 SHA512:
-  metadata.gz: ed938fa078cb982b23efe5da8a1a0fcb6d367534e7eeb342b2ce4853bf82cb7f0bb7998fd1a0bfa3bc3fd61a9f5d049717e245235dcf3df5a072be0b54eb9c2b
-  data.tar.gz: c1f2db9dbde2a28321deee3b4186c08eb06f3da1f231f652d1822692e67e479e7be3a7b22daada77bc8130e9d6e016027b8470f9c84eceb827f4e06a17ac6d12
+  metadata.gz: fc9bb3ac74a50d3103c0acc538ec7ef171324427f78ef62eadcc1afc5a2def15f3ef7063f3f74150aad20f3e93a406d2c3f2a0ca19aa63cff824d1e684637e19
+  data.tar.gz: cca0048ff2181b6346e0628dcb9710ee83937d2b7a1014c0133b0b7747b2b5328e73d29d4164daedeaddde9cb6e879a52a73646e62ca95e0c8845c2dc044d3d0

data/README.adoc

@@ -25,7 +25,7 @@ def fib(n)
   fib(n - 1) + fib(n - 2)
 end
 
-GvlTracing.start("example1.json") do
+GvlTracing.start("example1.json", os_threads_view_enabled: false) do
   Thread.new { sleep(0.05) while true }
 
   sleep(0.05)
@@ -68,6 +68,12 @@ This gem only provides a single module (`GvlTracing`) with methods:
 
 The resulting traces can be analyzed by going to https://ui.perfetto.dev/[Perfetto UI].
 
+== Experimental features
+
+1. Sleep tracking: Add `require 'gvl_tracing/sleep_tracking'` to add a more specific `sleeping` state for sleeps (which are otherwise rendered as `waiting` without this feature)
+
+2. OS threads view: Pass in `os_threads_view_enabled: true` to `GvlTracing.start` to also render a view of Ruby thread activity from the OS native threads point-of-view. This is useful when using M:N thread scheduling, which is used on Ruby 3.3+ Ractors, and when using the `RUBY_MN_THREADS=1` setting.
+
 == Tips
 
 You can "embed" links to the perfetto UI which trigger loading of a trace by following the instructions on https://perfetto.dev/docs/visualization/deep-linking-to-perfetto-ui .

data/ext/gvl_tracing_native_extension/gvl_tracing.c

@@ -31,7 +31,8 @@
 #include <inttypes.h>
 #include <stdbool.h>
 #include <sys/types.h>
-#include <threads.h>
+#include <pthread.h>
+#include <stdint.h>
 
 #include "extconf.h"
 
@@ -56,6 +57,10 @@
   #define RUBY_3_2
 #endif
 
+// For the OS threads view, we emit data as if it was for another pid so it gets grouped separately in perfetto.
+// This is a really big hack, but I couldn't think of a better way?
+#define OS_THREADS_VIEW_PID (INT64_C(0))
+
 typedef struct {
   bool initialized;
   int32_t current_thread_serial;
@@ -77,13 +82,14 @@ static VALUE gc_tracepoint = Qnil;
 #pragma GCC diagnostic ignored "-Wunused-variable"
 static int thread_storage_key = 0;
 static VALUE all_seen_threads = Qnil;
-static mtx_t all_seen_threads_mutex;
+static pthread_mutex_t all_seen_threads_mutex = PTHREAD_MUTEX_INITIALIZER;
+static bool os_threads_view_enabled;
 
 static VALUE tracing_init_local_storage(VALUE, VALUE);
-static VALUE tracing_start(VALUE _self, VALUE output_path);
+static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path, VALUE os_threads_view_enabled_arg);
 static VALUE tracing_stop(VALUE _self);
 static double timestamp_microseconds(void);
-static void render_event(thread_local_state *, const char *event_name);
+static double render_event(thread_local_state *, const char *event_name);
 static void on_thread_event(rb_event_flag_t event, const rb_internal_thread_event_data_t *_unused1, void *_unused2);
 static void on_gc_event(VALUE tpval, void *_unused1);
 static VALUE mark_sleeping(VALUE _self);
@@ -92,6 +98,9 @@ static void thread_local_state_mark(void *data);
 static inline int32_t thread_id_for(thread_local_state *state);
 static VALUE ruby_thread_id_for(UNUSED_ARG VALUE _self, VALUE thread);
 static VALUE trim_all_seen_threads(UNUSED_ARG VALUE _self);
+static void render_os_thread_event(thread_local_state *state, double now_microseconds);
+static void finish_previous_os_thread_event(double now_microseconds);
+static inline uint32_t current_native_thread_id(void);
 
 #pragma GCC diagnostic ignored "-Wunused-const-variable"
 static const rb_data_type_t thread_local_state_type = {
@@ -128,12 +137,10 @@ void Init_gvl_tracing_native_extension(void) {
 
   all_seen_threads = rb_ary_new();
 
-  if (mtx_init(&all_seen_threads_mutex, mtx_plain) != thrd_success) rb_raise(rb_eRuntimeError, "Failed to initialize GvlTracing mutex");
-
   VALUE gvl_tracing_module = rb_define_module("GvlTracing");
 
   rb_define_singleton_method(gvl_tracing_module, "_init_local_storage", tracing_init_local_storage, 1);
-  rb_define_singleton_method(gvl_tracing_module, "_start", tracing_start, 1);
+  rb_define_singleton_method(gvl_tracing_module, "_start", tracing_start, 2);
   rb_define_singleton_method(gvl_tracing_module, "_stop", tracing_stop, 0);
   rb_define_singleton_method(gvl_tracing_module, "mark_sleeping", mark_sleeping, 0);
   rb_define_singleton_method(gvl_tracing_module, "_thread_id_for", ruby_thread_id_for, 1);
@@ -145,19 +152,7 @@ static inline void initialize_thread_local_state(thread_local_state *state) {
   state->current_thread_serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1);
 
   #ifdef RUBY_3_2
-    uint32_t native_thread_id = 0;
-
-    #ifdef HAVE_PTHREAD_THREADID_NP
-      pthread_threadid_np(pthread_self(), &native_thread_id);
-    #elif HAVE_GETTID
-      native_thread_id = gettid();
-    #else
-      // Note: We could use the current_thread_serial as a crappy fallback, but this would make getting thread names
-      // not work very well
-      #error No native thread id available?
-    #endif
-
-    state->native_thread_id = native_thread_id;
+    state->native_thread_id = current_native_thread_id();
   #endif
 }
 
@@ -171,8 +166,9 @@ static VALUE tracing_init_local_storage(UNUSED_ARG VALUE _self, VALUE threads) {
   return Qtrue;
 }
 
-static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path) {
+static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path, VALUE os_threads_view_enabled_arg) {
   Check_Type(output_path, T_STRING);
+  if (os_threads_view_enabled_arg != Qtrue && os_threads_view_enabled_arg != Qfalse) rb_raise(rb_eArgError, "os_threads_view_enabled must be true/false");
 
   trim_all_seen_threads(Qnil);
 
@@ -180,13 +176,26 @@ static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path) {
   output_file = fopen(StringValuePtr(output_path), "w");
   if (output_file == NULL) rb_syserr_fail(errno, "Failed to open GvlTracing output file");
 
-  fprintf(output_file, "[\n");
-
   thread_local_state *state = GT_CURRENT_THREAD_LOCAL_STATE();
   started_tracing_at_microseconds = timestamp_microseconds();
   process_id = getpid();
+  os_threads_view_enabled = (os_threads_view_enabled_arg == Qtrue);
+
+  VALUE ruby_version = rb_const_get(rb_cObject, rb_intern("RUBY_VERSION"));
+  Check_Type(ruby_version, T_STRING);
+
+  fprintf(output_file, "[\n");
+  fprintf(output_file,
+    "  {\"ph\": \"M\", \"pid\": %"PRId64", \"name\": \"process_name\", \"args\": {\"name\": \"Ruby threads view (%s)\"}},\n",
+    process_id, StringValuePtr(ruby_version)
+  );
 
-  render_event(state, "started_tracing");
+  double now_microseconds = render_event(state, "started_tracing");
+
+  if (os_threads_view_enabled) {
+    fprintf(output_file, "  {\"ph\": \"M\", \"pid\": %"PRId64", \"name\": \"process_name\", \"args\": {\"name\": \"OS threads view\"}},\n", OS_THREADS_VIEW_PID);
+    render_os_thread_event(state, now_microseconds);
+  }
 
   current_hook = rb_internal_thread_add_event_hook(
     on_thread_event,
@@ -215,7 +224,9 @@ static VALUE tracing_stop(UNUSED_ARG VALUE _self) {
   rb_tracepoint_disable(gc_tracepoint);
   gc_tracepoint = Qnil;
 
-  render_event(state, "stopped_tracing");
+  double now_microseconds = render_event(state, "stopped_tracing");
+  if (os_threads_view_enabled) finish_previous_os_thread_event(now_microseconds);
+
   // closing the json syntax in the output file is handled in GvlTracing.stop code
 
   if (fclose(output_file) != 0) rb_syserr_fail(errno, "Failed to close GvlTracing output file");
@@ -237,7 +248,7 @@ static double timestamp_microseconds(void) {
 
 // Render output using trace event format for perfetto:
 // https://chromium.googlesource.com/catapult/+/refs/heads/main/docs/trace-event-format.md
-static void render_event(thread_local_state *state, const char *event_name) {
+static double render_event(thread_local_state *state, const char *event_name) {
   // Event data
   double now_microseconds = timestamp_microseconds() - started_tracing_at_microseconds;
 
@@ -258,6 +269,8 @@ static void render_event(thread_local_state *state, const char *event_name) {
     // Args for second line
     process_id, thread_id_for(state), now_microseconds, event_name
   );
+
+  return now_microseconds;
 }
 
 static void on_thread_event(rb_event_flag_t event_id, const rb_internal_thread_event_data_t *event_data, UNUSED_ARG void *_unused2) {
@@ -295,7 +308,15 @@ static void on_thread_event(rb_event_flag_t event_id, const rb_internal_thread_e
     case RUBY_INTERNAL_THREAD_EVENT_STARTED:   event_name = "started";   break;
     case RUBY_INTERNAL_THREAD_EVENT_EXITED:    event_name = "died";      break;
   };
-  render_event(state, event_name);
+  double now_microseconds = render_event(state, event_name);
+
+  if (os_threads_view_enabled) {
+    if (event_id == RUBY_INTERNAL_THREAD_EVENT_RESUMED) {
+      render_os_thread_event(state, now_microseconds);
+    } else if (event_id == RUBY_INTERNAL_THREAD_EVENT_SUSPENDED || event_id == RUBY_INTERNAL_THREAD_EVENT_EXITED) {
+      finish_previous_os_thread_event(now_microseconds);
+    }
+  }
 }
 
 static void on_gc_event(VALUE tpval, UNUSED_ARG void *_unused1) {
@@ -324,6 +345,16 @@ static void thread_local_state_mark(void *data) {
   rb_gc_mark(state->thread); // Marking thread to make sure it stays pinned
 }
 
+static inline void all_seen_threads_mutex_lock(void) {
+  int error = pthread_mutex_lock(&all_seen_threads_mutex);
+  if (error) rb_syserr_fail(error, "Failed to lock GvlTracing mutex");
+}
+
+static inline void all_seen_threads_mutex_unlock(void) {
+  int error = pthread_mutex_unlock(&all_seen_threads_mutex);
+  if (error) rb_syserr_fail(error, "Failed to unlock GvlTracing mutex");
+}
+
 #ifdef RUBY_3_3_PLUS
   static inline thread_local_state *GT_LOCAL_STATE(VALUE thread, bool allocate) {
     thread_local_state *state = rb_internal_thread_specific_get(thread, thread_storage_key);
@@ -338,9 +369,9 @@ static void thread_local_state_mark(void *data) {
       // Keep thread around, to be able to extract names at the end
       // We grab a lock here since thread creation can happen in multiple Ractors, and we want to make sure only one
       // of them is mutating the array at a time. @ivoanjo: I think this is enough to make this safe....?
-      if (mtx_lock(&all_seen_threads_mutex) != thrd_success) rb_raise(rb_eRuntimeError, "Failed to lock GvlTracing mutex");
+      all_seen_threads_mutex_lock();
       rb_ary_push(all_seen_threads, thread);
-      if (mtx_unlock(&all_seen_threads_mutex) != thrd_success) rb_raise(rb_eRuntimeError, "Failed to unlock GvlTracing mutex");
+      all_seen_threads_mutex_unlock();
     }
     return state;
   }
@@ -385,7 +416,7 @@ static VALUE ruby_thread_id_for(UNUSED_ARG VALUE _self, VALUE thread) {
 
 // Can only be called while GvlTracing is not active + while holding the GVL
 static VALUE trim_all_seen_threads(UNUSED_ARG VALUE _self) {
-  if (mtx_lock(&all_seen_threads_mutex) != thrd_success) rb_raise(rb_eRuntimeError, "Failed to lock GvlTracing mutex");
+  all_seen_threads_mutex_lock();
 
   VALUE alive_threads = rb_ary_new();
 
@@ -398,6 +429,51 @@ static VALUE trim_all_seen_threads(UNUSED_ARG VALUE _self) {
 
   rb_ary_replace(all_seen_threads, alive_threads);
 
-  if (mtx_unlock(&all_seen_threads_mutex) != thrd_success) rb_raise(rb_eRuntimeError, "Failed to unlock GvlTracing mutex");
+  all_seen_threads_mutex_unlock();
   return Qtrue;
 }
+
+// Creates an event that follows the current native thread. Note that this assumes that whatever event
+// made us call `render_os_thread_event` is an event about the current (native) thread; if the event is not about the
+// current thread, the results will be incorrect.
+static void render_os_thread_event(thread_local_state *state, double now_microseconds) {
+  finish_previous_os_thread_event(now_microseconds);
+
+  // Hack: If we name threads as "Thread N", perfetto seems to color them all with the same color, which looks awful.
+  // I did not check the code, but in practice perfetto seems to be doing some kind of hashing based only on regular
+  // chars, so here we append a different letter to each thread to cause the color hashing to differ.
+  char color_suffix_hack = ('a' + (thread_id_for(state) % 26));
+
+  fprintf(output_file,
+    "  {\"ph\": \"B\", \"pid\": %"PRId64", \"tid\": %u, \"ts\": %f, \"name\": \"Thread %d (%c)\"},\n",
+    OS_THREADS_VIEW_PID, current_native_thread_id(), now_microseconds, thread_id_for(state), color_suffix_hack
+  );
+}
+
+static void finish_previous_os_thread_event(double now_microseconds) {
+  fprintf(output_file,
+    "  {\"ph\": \"E\", \"pid\": %"PRId64", \"tid\": %u, \"ts\": %f},\n",
+    OS_THREADS_VIEW_PID, current_native_thread_id(), now_microseconds
+  );
+}
+
+static inline uint32_t current_native_thread_id(void) {
+  uint32_t native_thread_id = 0;
+
+  #ifdef HAVE_PTHREAD_THREADID_NP
+    uint64_t full_native_thread_id;
+    pthread_threadid_np(pthread_self(), &full_native_thread_id);
+    // Note: `pthread_threadid_np` is declared as taking in a `uint64_t` but I don't think macOS uses such really
+    // high thread ids, and anyway perfetto doesn't like full 64-bit ids for threads so let's go with a simplification
+    // for now.
+    native_thread_id = (uint32_t) full_native_thread_id;
+  #elif HAVE_GETTID
+    native_thread_id = gettid();
+  #else
+    // Note: We could use a native thread-local crappy fallback, but I think the two above alternatives are available
+    // on all OSs that support the GVL tracing API.
+    #error No native thread id available?
+  #endif
+
+  return native_thread_id;
+}

data/gems.rb

@@ -9,6 +9,7 @@ gem "rake-compiler", "~> 1.2"
 gem "pry"
 gem "pry-byebug"
 gem "rspec"
-gem "standard", "~> 1.33"
+gem "standard", "~> 1.41"
 gem "concurrent-ruby"
 gem "benchmark-ips", "~> 2.13"
+gem "rubocop", ">= 1.66.0"

data/lib/gvl-tracing.rb

@@ -34,8 +34,8 @@ module GvlTracing
     private :_start
     private :_stop
 
-    def start(file)
-      _start(file)
+    def start(file, os_threads_view_enabled: false)
+      _start(file, os_threads_view_enabled)
       _init_local_storage(Thread.list)
       @path = file
 

data/lib/gvl_tracing/sleep_tracking.rb

@@ -5,7 +5,7 @@
 module GvlTracing::SleepTracking
   def sleep(...)
     GvlTracing.mark_sleeping
-    super(...)
+    super
   end
 end
 

data/lib/gvl_tracing/version.rb

@@ -26,5 +26,5 @@
 # frozen_string_literal: true
 
 module GvlTracing
-  VERSION = "1.5.1"
+  VERSION = "1.6.0"
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: gvl-tracing
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Ivo Anjo
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-29 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
 - ivo@ivoanjo.me
 executables: []
@@ -37,7 +37,7 @@ homepage: https://github.com/ivoanjo/gvl-tracing
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -53,8 +53,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key: 
+rubygems_version: 3.5.11
+signing_key:
 specification_version: 4
 summary: Get a timeline view of Global VM Lock usage in your Ruby app
 test_files: []