gvl-tracing 1.5.2 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f2845379f02ef9e8cd317a293b0c24c92dbd9aebad15ccb9b378ea1c41cf894
-  data.tar.gz: ce639fb1bffb10262e1dee5f5bf7a80f4425d9db7b690a494b6cefedae507265
+  metadata.gz: 2bf28c6665bf4afa02c13299947bb65d7c62c3af783858afdc3a5383e334fa47
+  data.tar.gz: b1999e7e17d56e76ba732962708c14979e2690cd78204946d2f05e7146b05725
 SHA512:
-  metadata.gz: '01659c02734923610f4c771afcd1a863de369781a7c21cf42c46e5d956f897c65643f0ae6a5a1eebb7fcb81895556271f57f7e164b3ac3581177509d31381db7'
-  data.tar.gz: f0e0894e7d3dcc5c7f00485b8e741be31f832142ef0436524f26e5996da9f901dc17b46080b26f2ecf5bcfc0ca37b6bdaae48b014eaeb00518dfa56b6c892167
+  metadata.gz: feb5fb173e07343af18c929608b20b2463fc0b676565b926242585f3fed31991e34ab5f549d7cbfe0e26c883d4a1e6eebf1d8da9e1586f0fa63d0890cdd3b108
+  data.tar.gz: 615226e345e230b989d7738754f02a058935d67679b4ca16c610cbbb6fe34f82aff6de3299d9548af579a30d55bccaedf25cd52d4f631023cff93a683fe3c8bb

data/README.adoc

@@ -25,7 +25,7 @@ def fib(n)
   fib(n - 1) + fib(n - 2)
 end
 
-GvlTracing.start("example1.json") do
+GvlTracing.start("example1.json", os_threads_view_enabled: false) do
   Thread.new { sleep(0.05) while true }
 
   sleep(0.05)
@@ -68,6 +68,12 @@ This gem only provides a single module (`GvlTracing`) with methods:
 
 The resulting traces can be analyzed by going to https://ui.perfetto.dev/[Perfetto UI].
 
+== Experimental features
+
+1. Sleep tracking: Add `require 'gvl_tracing/sleep_tracking'` to add a more specific `sleeping` state for sleeps (which are otherwise rendered as `waiting` without this feature)
+
+2. OS threads view: Pass in `os_threads_view_enabled: true` to `GvlTracing.start` to also render a view of Ruby thread activity from the OS native threads point-of-view. This is useful when using M:N thread scheduling, which is used on Ruby 3.3+ Ractors, and when using the `RUBY_MN_THREADS=1` setting.
+
 == Tips
 
 You can "embed" links to the perfetto UI which trigger loading of a trace by following the instructions on https://perfetto.dev/docs/visualization/deep-linking-to-perfetto-ui .

data/ext/gvl_tracing_native_extension/gvl_tracing.c

@@ -32,6 +32,7 @@
 #include <stdbool.h>
 #include <sys/types.h>
 #include <pthread.h>
+#include <stdint.h>
 
 #include "extconf.h"
 
@@ -56,6 +57,10 @@
   #define RUBY_3_2
 #endif
 
+// For the OS threads view, we emit data as if it was for another pid so it gets grouped separately in perfetto.
+// This is a really big hack, but I couldn't think of a better way?
+#define OS_THREADS_VIEW_PID (INT64_C(0))
+
 typedef struct {
   bool initialized;
   int32_t current_thread_serial;
@@ -78,20 +83,27 @@ static VALUE gc_tracepoint = Qnil;
 static int thread_storage_key = 0;
 static VALUE all_seen_threads = Qnil;
 static pthread_mutex_t all_seen_threads_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t output_mutex = PTHREAD_MUTEX_INITIALIZER;
+static bool os_threads_view_enabled;
 
 static VALUE tracing_init_local_storage(VALUE, VALUE);
-static VALUE tracing_start(VALUE _self, VALUE output_path);
+static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path, VALUE os_threads_view_enabled_arg);
 static VALUE tracing_stop(VALUE _self);
 static double timestamp_microseconds(void);
-static void render_event(thread_local_state *, const char *event_name);
+static double render_event(thread_local_state *, const char *event_name);
 static void on_thread_event(rb_event_flag_t event, const rb_internal_thread_event_data_t *_unused1, void *_unused2);
 static void on_gc_event(VALUE tpval, void *_unused1);
 static VALUE mark_sleeping(VALUE _self);
 static size_t thread_local_state_memsize(UNUSED_ARG const void *_unused);
 static void thread_local_state_mark(void *data);
+static inline void output_mutex_lock(void);
+static inline void output_mutex_unlock(void);
 static inline int32_t thread_id_for(thread_local_state *state);
 static VALUE ruby_thread_id_for(UNUSED_ARG VALUE _self, VALUE thread);
 static VALUE trim_all_seen_threads(UNUSED_ARG VALUE _self);
+static void render_os_thread_event(thread_local_state *state, double now_microseconds);
+static void finish_previous_os_thread_event(double now_microseconds);
+static inline uint32_t current_native_thread_id(void);
 
 #pragma GCC diagnostic ignored "-Wunused-const-variable"
 static const rb_data_type_t thread_local_state_type = {
@@ -131,7 +143,7 @@ void Init_gvl_tracing_native_extension(void) {
   VALUE gvl_tracing_module = rb_define_module("GvlTracing");
 
   rb_define_singleton_method(gvl_tracing_module, "_init_local_storage", tracing_init_local_storage, 1);
-  rb_define_singleton_method(gvl_tracing_module, "_start", tracing_start, 1);
+  rb_define_singleton_method(gvl_tracing_module, "_start", tracing_start, 2);
   rb_define_singleton_method(gvl_tracing_module, "_stop", tracing_stop, 0);
   rb_define_singleton_method(gvl_tracing_module, "mark_sleeping", mark_sleeping, 0);
   rb_define_singleton_method(gvl_tracing_module, "_thread_id_for", ruby_thread_id_for, 1);
@@ -143,24 +155,7 @@ static inline void initialize_thread_local_state(thread_local_state *state) {
   state->current_thread_serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1);
 
   #ifdef RUBY_3_2
-    uint32_t native_thread_id = 0;
-
-    #ifdef HAVE_PTHREAD_THREADID_NP
-      uint64_t full_native_thread_id;
-      pthread_threadid_np(pthread_self(), &full_native_thread_id);
-      // Note: `pthread_threadid_np` is declared as taking in a `uint64_t` but I don't think macOS uses such really
-      // high thread ids, and anyway perfetto doesn't like full 64-bit ids for threads so let's go with a simplification
-      // for now.
-      native_thread_id = (uint32_t) full_native_thread_id;
-    #elif HAVE_GETTID
-      native_thread_id = gettid();
-    #else
-      // Note: We could use the current_thread_serial as a crappy fallback, but this would make getting thread names
-      // not work very well
-      #error No native thread id available?
-    #endif
-
-    state->native_thread_id = native_thread_id;
+    state->native_thread_id = current_native_thread_id();
   #endif
 }
 
@@ -174,8 +169,9 @@ static VALUE tracing_init_local_storage(UNUSED_ARG VALUE _self, VALUE threads) {
   return Qtrue;
 }
 
-static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path) {
+static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path, VALUE os_threads_view_enabled_arg) {
   Check_Type(output_path, T_STRING);
+  if (os_threads_view_enabled_arg != Qtrue && os_threads_view_enabled_arg != Qfalse) rb_raise(rb_eArgError, "os_threads_view_enabled must be true/false");
 
   trim_all_seen_threads(Qnil);
 
@@ -183,13 +179,26 @@ static VALUE tracing_start(UNUSED_ARG VALUE _self, VALUE output_path) {
   output_file = fopen(StringValuePtr(output_path), "w");
   if (output_file == NULL) rb_syserr_fail(errno, "Failed to open GvlTracing output file");
 
-  fprintf(output_file, "[\n");
-
   thread_local_state *state = GT_CURRENT_THREAD_LOCAL_STATE();
   started_tracing_at_microseconds = timestamp_microseconds();
   process_id = getpid();
+  os_threads_view_enabled = (os_threads_view_enabled_arg == Qtrue);
+
+  VALUE ruby_version = rb_const_get(rb_cObject, rb_intern("RUBY_VERSION"));
+  Check_Type(ruby_version, T_STRING);
+
+  fprintf(output_file, "[\n");
+  fprintf(output_file,
+    "  {\"ph\": \"M\", \"pid\": %"PRId64", \"name\": \"process_name\", \"args\": {\"name\": \"Ruby threads view (%s)\"}},\n",
+    process_id, StringValuePtr(ruby_version)
+  );
+
+  double now_microseconds = render_event(state, "started_tracing");
 
-  render_event(state, "started_tracing");
+  if (os_threads_view_enabled) {
+    fprintf(output_file, "  {\"ph\": \"M\", \"pid\": %"PRId64", \"name\": \"process_name\", \"args\": {\"name\": \"OS threads view\"}},\n", OS_THREADS_VIEW_PID);
+    render_os_thread_event(state, now_microseconds);
+  }
 
   current_hook = rb_internal_thread_add_event_hook(
     on_thread_event,
@@ -218,7 +227,9 @@ static VALUE tracing_stop(UNUSED_ARG VALUE _self) {
   rb_tracepoint_disable(gc_tracepoint);
   gc_tracepoint = Qnil;
 
-  render_event(state, "stopped_tracing");
+  double now_microseconds = render_event(state, "stopped_tracing");
+  if (os_threads_view_enabled) finish_previous_os_thread_event(now_microseconds);
+
   // closing the json syntax in the output file is handled in GvlTracing.stop code
 
   if (fclose(output_file) != 0) rb_syserr_fail(errno, "Failed to close GvlTracing output file");
@@ -240,7 +251,7 @@ static double timestamp_microseconds(void) {
 
 // Render output using trace event format for perfetto:
 // https://chromium.googlesource.com/catapult/+/refs/heads/main/docs/trace-event-format.md
-static void render_event(thread_local_state *state, const char *event_name) {
+static double render_event(thread_local_state *state, const char *event_name) {
   // Event data
   double now_microseconds = timestamp_microseconds() - started_tracing_at_microseconds;
 
@@ -251,6 +262,7 @@ static void render_event(thread_local_state *state, const char *event_name) {
   // Important note: We've observed some rendering issues in perfetto if the tid or pid are numbers that are "too big",
   // see https://github.com/ivoanjo/gvl-tracing/pull/4#issuecomment-1196463364 for an example.
 
+  output_mutex_lock();
   fprintf(output_file,
     // Finish previous duration
     "  {\"ph\": \"E\", \"pid\": %"PRId64", \"tid\": %d, \"ts\": %f},\n" \
@@ -261,6 +273,9 @@ static void render_event(thread_local_state *state, const char *event_name) {
     // Args for second line
     process_id, thread_id_for(state), now_microseconds, event_name
   );
+  output_mutex_unlock();
+
+  return now_microseconds;
 }
 
 static void on_thread_event(rb_event_flag_t event_id, const rb_internal_thread_event_data_t *event_data, UNUSED_ARG void *_unused2) {
@@ -298,7 +313,15 @@ static void on_thread_event(rb_event_flag_t event_id, const rb_internal_thread_e
     case RUBY_INTERNAL_THREAD_EVENT_STARTED:   event_name = "started";   break;
     case RUBY_INTERNAL_THREAD_EVENT_EXITED:    event_name = "died";      break;
   };
-  render_event(state, event_name);
+  double now_microseconds = render_event(state, event_name);
+
+  if (os_threads_view_enabled) {
+    if (event_id == RUBY_INTERNAL_THREAD_EVENT_RESUMED) {
+      render_os_thread_event(state, now_microseconds);
+    } else if (event_id == RUBY_INTERNAL_THREAD_EVENT_SUSPENDED || event_id == RUBY_INTERNAL_THREAD_EVENT_EXITED) {
+      finish_previous_os_thread_event(now_microseconds);
+    }
+  }
 }
 
 static void on_gc_event(VALUE tpval, UNUSED_ARG void *_unused1) {
@@ -337,6 +360,18 @@ static inline void all_seen_threads_mutex_unlock(void) {
   if (error) rb_syserr_fail(error, "Failed to unlock GvlTracing mutex");
 }
 
+static inline void output_mutex_lock(void) {
+  int error = pthread_mutex_lock(&output_mutex);
+  // Can't raise exceptions on error since it gets used from outside the GVL
+  if (error) fprintf(stderr, "Failed to lock the GvlTracing output_mutex");
+}
+
+static inline void output_mutex_unlock(void) {
+  int error = pthread_mutex_unlock(&output_mutex);
+  // Can't raise exceptions on error since it gets used from outside the GVL
+  if (error) fprintf(stderr, "Failed to unlock the GvlTracing output_mutex");
+}
+
 #ifdef RUBY_3_3_PLUS
   static inline thread_local_state *GT_LOCAL_STATE(VALUE thread, bool allocate) {
     thread_local_state *state = rb_internal_thread_specific_get(thread, thread_storage_key);
@@ -414,3 +449,52 @@ static VALUE trim_all_seen_threads(UNUSED_ARG VALUE _self) {
   all_seen_threads_mutex_unlock();
   return Qtrue;
 }
+
+// Creates an event that follows the current native thread. Note that this assumes that whatever event
+// made us call `render_os_thread_event` is an event about the current (native) thread; if the event is not about the
+// current thread, the results will be incorrect.
+static void render_os_thread_event(thread_local_state *state, double now_microseconds) {
+  finish_previous_os_thread_event(now_microseconds);
+
+  // Hack: If we name threads as "Thread N", perfetto seems to color them all with the same color, which looks awful.
+  // I did not check the code, but in practice perfetto seems to be doing some kind of hashing based only on regular
+  // chars, so here we append a different letter to each thread to cause the color hashing to differ.
+  char color_suffix_hack = ('a' + (thread_id_for(state) % 26));
+
+  output_mutex_lock();
+  fprintf(output_file,
+    "  {\"ph\": \"B\", \"pid\": %"PRId64", \"tid\": %u, \"ts\": %f, \"name\": \"Thread %d (%c)\"},\n",
+    OS_THREADS_VIEW_PID, current_native_thread_id(), now_microseconds, thread_id_for(state), color_suffix_hack
+  );
+  output_mutex_unlock();
+}
+
+static void finish_previous_os_thread_event(double now_microseconds) {
+  output_mutex_lock();
+  fprintf(output_file,
+    "  {\"ph\": \"E\", \"pid\": %"PRId64", \"tid\": %u, \"ts\": %f},\n",
+    OS_THREADS_VIEW_PID, current_native_thread_id(), now_microseconds
+  );
+  output_mutex_unlock();
+}
+
+static inline uint32_t current_native_thread_id(void) {
+  uint32_t native_thread_id = 0;
+
+  #ifdef HAVE_PTHREAD_THREADID_NP
+    uint64_t full_native_thread_id;
+    pthread_threadid_np(pthread_self(), &full_native_thread_id);
+    // Note: `pthread_threadid_np` is declared as taking in a `uint64_t` but I don't think macOS uses such really
+    // high thread ids, and anyway perfetto doesn't like full 64-bit ids for threads so let's go with a simplification
+    // for now.
+    native_thread_id = (uint32_t) full_native_thread_id;
+  #elif HAVE_GETTID
+    native_thread_id = gettid();
+  #else
+    // Note: We could use a native thread-local crappy fallback, but I think the two above alternatives are available
+    // on all OSs that support the GVL tracing API.
+    #error No native thread id available?
+  #endif
+
+  return native_thread_id;
+}

data/gvl-tracing.gemspec

@@ -42,7 +42,8 @@ Gem::Specification.new do |spec|
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features|examples)/|\.(?:git|travis|circleci)|appveyor)})
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features|examples)/|\.(?:git|travis|circleci)|appveyor)}) ||
+        [".editorconfig", ".ruby-version", ".standard.yml", "gems.rb", "Rakefile"].include?(f)
     end
   end
   spec.require_paths = ["lib", "ext"]

data/lib/gvl-tracing.rb

@@ -34,8 +34,8 @@ module GvlTracing
     private :_start
     private :_stop
 
-    def start(file)
-      _start(file)
+    def start(file, os_threads_view_enabled: false)
+      _start(file, os_threads_view_enabled)
       _init_local_storage(Thread.list)
       @path = file
 

data/lib/gvl_tracing/sleep_tracking.rb

@@ -5,7 +5,7 @@
 module GvlTracing::SleepTracking
   def sleep(...)
     GvlTracing.mark_sleeping
-    super(...)
+    super
   end
 end
 

data/lib/gvl_tracing/version.rb

@@ -26,5 +26,5 @@
 # frozen_string_literal: true
 
 module GvlTracing
-  VERSION = "1.5.2"
+  VERSION = "1.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gvl-tracing
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.6.1
 platform: ruby
 authors:
 - Ivo Anjo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-04-06 00:00:00.000000000 Z
+date: 2024-12-08 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -18,16 +18,11 @@ extensions:
 - ext/gvl_tracing_native_extension/extconf.rb
 extra_rdoc_files: []
 files:
-- ".editorconfig"
-- ".ruby-version"
-- ".standard.yml"
 - CODE_OF_CONDUCT.adoc
 - LICENSE
 - README.adoc
-- Rakefile
 - ext/gvl_tracing_native_extension/extconf.rb
 - ext/gvl_tracing_native_extension/gvl_tracing.c
-- gems.rb
 - gvl-tracing.gemspec
 - lib/gvl-tracing.rb
 - lib/gvl_tracing/sleep_tracking.rb
@@ -53,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: Get a timeline view of Global VM Lock usage in your Ruby app

data/.editorconfig

@@ -1,22 +0,0 @@
-# EditorConfig is awesome: https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# Unix-style newlines with a newline ending every file
-[*]
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-[*.h]
-indent_style = space
-indent_size = 2
-
-[*.c]
-indent_style = space
-indent_size = 2
-
-[*.yml]
-indent_style = space
-indent_size = 2

data/.ruby-version

@@ -1 +0,0 @@
-ruby-3.2.2

data/.standard.yml

@@ -1,9 +0,0 @@
-ruby_version: 3.2
-
-ignore:
-  - 'examples/example1.rb':
-    - Style/InfiniteLoop
-  - 'examples/example5.rb':
-    - Style/InfiniteLoop
-  - 'lib/gvl_tracing/sleep_tracking.rb':
-    - Style/MixinUsage

data/Rakefile

@@ -1,36 +0,0 @@
-# gvl-tracing: Ruby gem for getting a timelinew view of GVL usage
-# Copyright (c) 2022 Ivo Anjo <ivo@ivoanjo.me>
-#
-# This file is part of gvl-tracing.
-#
-# MIT License
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-# frozen_string_literal: true
-
-require "bundler/gem_tasks"
-require "standard/rake"
-require "rake/extensiontask"
-require "rspec/core/rake_task"
-
-Rake::ExtensionTask.new("gvl_tracing_native_extension")
-RSpec::Core::RakeTask.new(:spec)
-
-task default: [:compile, :"standard:fix", :spec]

data/gems.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gemspec
-
-gem "rake", "~> 13.0"
-gem "rake-compiler", "~> 1.2"
-gem "pry"
-gem "pry-byebug"
-gem "rspec"
-gem "standard", "~> 1.33"
-gem "concurrent-ruby"
-gem "benchmark-ips", "~> 2.13"