pf2 0.9.0 → 1.0.0.alpha1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a0dc2d338a482c56472ec6279cc4fd111c4263a540ab00872d57439a14244af
-  data.tar.gz: 56bdc8e81a1d4bccee07d7785eec713329c8ec4b9e659f2b494e12e8e016e60a
+  metadata.gz: 9f219b8aa5b5281a6ed662a085e00f82636ab52365d2b0f7a8d61a3e06bfc9c6
+  data.tar.gz: 162e1eae488afe17e33291f63f807642aa304d7df8069cb165e3a25ce6f10895
 SHA512:
-  metadata.gz: 43449c0433cfdc390aecf5335f2a8631d11a0a41ac3afc6fc469909e5fd28c45bde16de8fbcff5229c16551d26a2b47210ff2342dea5b190a47ca128e9656c80
-  data.tar.gz: e6f5f2481932100cd82504dcb3b24e71b123b1263a2588b584cf5e72e53817f2004c7ffe6e96ff12f9815975ddfeac679219468a1af9223f6b5a367b3ec0b5e6
+  metadata.gz: 37b6a1aa4f6ab0753d86983d76cbdfae4b3dfbb7464afb0388aa113d728725f09eabcabb2ae7c2ffadc5ab797be75675b5b27b465c20565d5d69f30291d4837f
+  data.tar.gz: 7b69aea55c8873cfd6e28fba3bdddd09325e4c73e1e3e44a5be48fcb9ad776b03e811c707fc049ad8ccc1590e7c011032be7f4053e3d15a5ba5b50c4344db656

data/README.md

@@ -116,7 +116,7 @@ Schedulers determine when to execute sample collection, based on configuration (
 
 The first is the `SignalScheduler`, based on POSIX timers. Pf2 will use this scheduler when possible. SignalScheduler creates a POSIX timer for each Ruby Thread (the underlying pthread to be more accurate) using `timer_create(2)`. This leaves the actual time-keeping to the OS, which is capable of tracking accurate per-thread CPU time usage.
 
-When the specified interval has arrived (the timer has _expired_), the OS delivers us a SIGALRM (note: Unlike `setitimer(2)`, `timer_create(2)` allows us to choose which signal to be delivered, and Pf2 uses SIGALRM regardless of time mode). This is why the scheduler is named SignalScheduler.
+When the specified interval has arrived (the timer has _expired_), the OS delivers us a SIGPROF signal. This is why the scheduler is named SignalScheduler.
 
 Signals are directed to Ruby Threads' underlying pthread, effectively "pausing" the Thread's activity. This routing is done using `SIGEV_THREAD_ID`, which is a Linux-only feature. Sample collection is done in the signal handler, which is expected to be more _accurate_, capturing the paused Thread's activity.
 
@@ -128,11 +128,16 @@ Another scheduler is the `TimerThreadScheduler`, which maintains a time-keeping
 
 This scheduler is wall-time only, and does not support CPU-time based profiling.
 
-Future Plans
+#### macOS Support
+
+On platforms where `timer_create()` is not supported (namely macOS), Pf2 falls back to `setitimer()`.
+
+
+Wishlist
 --------
 
-- Remove known limitations, if possible
-- Implement a "tracing" scheduler, using the C TracePoint API
+- [Flame Scopes](https://www.brendangregg.com/flamescope.html)
+- More unit/e2e tests
 - more
 
 Development

data/Rakefile

@@ -4,15 +4,9 @@ require 'minitest/test_task'
 
 task default: %i[]
 
-if ENV['PF2_PF2C'] == '1'
-  Rake::ExtensionTask.new 'pf2c' do |ext|
-    ext.name = 'pf2'
-    ext.lib_dir = 'lib/pf2'
-  end
-else
-  Rake::ExtensionTask.new 'pf2' do |ext|
-    ext.lib_dir = 'lib/pf2'
-  end
+Rake::ExtensionTask.new 'pf2' do |ext|
+  ext.name = 'pf2'
+  ext.lib_dir = 'lib/pf2'
 end
 
 Minitest::TestTask.create(:test) do |t|

data/doc/development.md

@@ -1,6 +1,12 @@
 Pf2 Development
 ===========
 
+Setup
+--------
+
+- `git submodule update --init`
+
+
 Releasing
 --------
 

data/ext/pf2/debug.h

@@ -0,0 +1,12 @@
+#ifndef PF2_DEBUG_H
+#define PF2_DEBUG_H
+
+#include <stdio.h>
+
+#ifdef PF2_DEBUG
+    #define PF2_DEBUG_LOG(format, ...) printf(format, ##__VA_ARGS__)
+#else
+    #define PF2_DEBUG_LOG(format, ...) ((void)0)
+#endif
+
+#endif // PF2_DEBUG_H

data/ext/pf2/extconf.rb

@@ -1,10 +1,27 @@
 require 'mkmf'
-require 'rb_sys/mkmf'
+require 'mini_portile2'
 
-abort 'missing rb_profile_thread_frames()' unless have_func 'rb_profile_thread_frames'
+libbacktrace = MiniPortile.new('libbacktrace', '1.0.0')
+libbacktrace.source_directory = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendor', 'libbacktrace'))
+libbacktrace.configure_options << 'CFLAGS=-fPIC'
+libbacktrace.cook
+libbacktrace.mkmf_config
 
-create_rust_makefile 'pf2/pf2' do |r|
-  if ENV['PF2_FEATURES']
-    r.features = ENV['PF2_FEATURES'].split(",")
-  end
+if !have_func('backtrace_full', 'backtrace.h')
+  raise 'libbacktrace has not been properly configured'
+end
+
+append_ldflags('-lrt') # for timer_create
+append_cflags('-fvisibility=hidden')
+append_cflags('-DPF2_DEBUG') if ENV['PF2_DEBUG'] == '1'
+
+# Check for timer functions
+have_timer_create = have_func('timer_create')
+have_setitimer = have_func('setitimer')
+
+if have_timer_create || have_setitimer
+  $srcs = Dir.glob("#{File.join(File.dirname(__FILE__), '*.c')}")
+  create_makefile 'pf2/pf2'
+else
+  raise 'Neither timer_create nor setitimer is available'
 end

data/ext/{pf2c → pf2}/sample.c

@@ -1,5 +1,6 @@
 #include <stdbool.h>
 #include <time.h>
+#include <pthread.h>
 
 #include <backtrace.h>
 #include <ruby.h>
@@ -17,11 +18,16 @@ static int backtrace_on_ok(void *data, uintptr_t pc);
 bool
 pf2_sample_capture(struct pf2_sample *sample)
 {
+    // Initialize sample
+    memset(sample, 0, sizeof(struct pf2_sample));
+
     // Record the current time
     struct timespec now;
     clock_gettime(CLOCK_MONOTONIC, &now);
     sample->timestamp_ns = (uint64_t)now.tv_sec * 1000000000ULL + (uint64_t)now.tv_nsec;
 
+    sample->context_pthread = pthread_self();
+
     // Obtain the current stack from Ruby
     sample->depth = rb_profile_frames(0, 200, sample->cmes, sample->linenos);
 

data/ext/{pf2c → pf2}/sample.h

@@ -1,11 +1,15 @@
 #ifndef PF2_SAMPLE_H
 #define PF2_SAMPLE_H
 
+#include <pthread.h>
+
 #include <ruby.h>
 
 extern const int PF2_SAMPLE_MAX_NATIVE_DEPTH;
 
 struct pf2_sample {
+    pthread_t context_pthread;
+
     int depth;
     VALUE cmes[200];
     int linenos[200];

data/ext/{pf2c → pf2}/serializer.c

@@ -83,7 +83,7 @@ pf2_ser_prepare(struct pf2_ser *serializer, struct pf2_session *session) {
         ensure_samples_capacity(serializer);
 
         struct pf2_ser_sample *ser_sample = &serializer->samples[serializer->samples_count++];
-        ser_sample->ruby_thread_id = 0; // TODO: Add thread ID support
+        ser_sample->ruby_thread_id = sample->context_pthread;
         ser_sample->elapsed_ns = sample->timestamp_ns - serializer->start_timestamp_ns;
 
         // Copy and process Ruby stack frames

data/ext/{pf2c → pf2}/session.c

@@ -1,11 +1,12 @@
 #include <bits/time.h>
+#include <pthread.h>
+#include <signal.h>
 #include <stdatomic.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <signal.h>
+#include <sys/time.h>
 #include <time.h>
-#include <pthread.h>
 
 #include <ruby.h>
 #include <ruby/debug.h>
@@ -14,10 +15,16 @@
 
 #include "backtrace_state.h"
 #include "configuration.h"
+#include "debug.h"
 #include "sample.h"
 #include "session.h"
 #include "serializer.h"
 
+#ifndef HAVE_TIMER_CREATE
+// Global session pointer for setitimer fallback
+static struct pf2_session *global_current_session = NULL;
+#endif
+
 static void *sample_collector_thread(void *arg);
 static void sigprof_handler(int sig, siginfo_t *info, void *ucontext);
 bool ensure_sample_capacity(struct pf2_session *session);
@@ -60,17 +67,28 @@ rb_pf2_session_start(VALUE self)
         rb_raise(rb_eRuntimeError, "Failed to spawn sample collector thread");
     }
 
-    // Configure signal handler
+    // Install signal handler for SIGPROF
     struct sigaction sa;
     sa.sa_sigaction = sigprof_handler;
     sigemptyset(&sa.sa_mask);
     sigaddset(&sa.sa_mask, SIGPROF); // Mask SIGPROFs when handler is running
     sa.sa_flags = SA_SIGINFO | SA_RESTART;
     if (sigaction(SIGPROF, &sa, NULL) == -1) {
-        rb_raise(rb_eRuntimeError, "Failed to install signal handler");
+        rb_raise(rb_eRuntimeError, "Failed to install SIGPROF handler");
     }
 
-    // Configure a timer to send SIGPROF every 10 ms of CPU time
+#ifndef HAVE_TIMER_CREATE
+    // Install signal handler for SIGALRM if using wall time mode with setitimer
+    if (session->configuration->time_mode != PF2_TIME_MODE_CPU_TIME) {
+        sigaddset(&sa.sa_mask, SIGALRM);
+        if (sigaction(SIGALRM, &sa, NULL) == -1) {
+            rb_raise(rb_eRuntimeError, "Failed to install SIGALRM handler");
+        }
+    }
+#endif
+
+#ifdef HAVE_TIMER_CREATE
+    // Configure a kernel timer to send SIGPROF periodically
     struct sigevent sev;
     sev.sigev_notify = SIGEV_SIGNAL;
     sev.sigev_signo = SIGPROF;
@@ -97,6 +115,30 @@ rb_pf2_session_start(VALUE self)
     if (timer_settime(session->timer, 0, &its, NULL) == -1) {
         rb_raise(rb_eRuntimeError, "Failed to start timer");
     }
+#else
+    // Use setitimer as fallback
+    // Some platforms (e.g. macOS) do not have timer_create(3).
+    // setitimer(3) can be used as a alternative, but has limited functionality.
+    global_current_session = session;
+
+    struct itimerval itv = {
+        .it_value = {
+            .tv_sec = 0,
+            .tv_usec = session->configuration->interval_ms * 1000,
+        },
+        .it_interval = {
+            .tv_sec = 0,
+            .tv_usec = session->configuration->interval_ms * 1000,
+        },
+    };
+    int which_timer = session->configuration->time_mode == PF2_TIME_MODE_CPU_TIME
+        ? ITIMER_PROF  // CPU time (sends SIGPROF)
+        : ITIMER_REAL; // Wall time (sends SIGALRM)
+
+    if (setitimer(which_timer, &itv, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to start timer");
+    }
+#endif
 
     return Qtrue;
 }
@@ -113,9 +155,7 @@ sample_collector_thread(void *arg)
             // Ensure we have capacity before adding a new sample
             if (!ensure_sample_capacity(session)) {
                 // Failed to expand buffer
-#ifdef PF2_DEBUG
-                printf("Failed to expand sample buffer. Dropping sample\n");
-#endif
+                PF2_DEBUG_LOG("Failed to expand sample buffer. Dropping sample\n");
                 break;
             }
 
@@ -140,31 +180,30 @@ sigprof_handler(int sig, siginfo_t *info, void *ucontext)
     clock_gettime(CLOCK_MONOTONIC, &sig_start_time);
 #endif
 
-    struct pf2_session *session = info->si_value.sival_ptr;
+    struct pf2_session *session;
+#ifdef HAVE_TIMER_CREATE
+    session = info->si_value.sival_ptr;
+#else
+    session = global_current_session;
+#endif
 
     // If garbage collection is in progress, don't collect samples.
     if (atomic_load_explicit(&session->is_marking, memory_order_acquire)) {
-#ifdef PF2_DEBUG
-        printf("Dropping sample: Garbage collection is in progress\n");
-#endif
+        PF2_DEBUG_LOG("Dropping sample: Garbage collection is in progress\n");
         return;
     }
 
-    struct pf2_sample sample = { 0 };
+    struct pf2_sample sample;
 
     if (pf2_sample_capture(&sample) == false) {
-#ifdef PF2_DEBUG
-        printf("Dropping sample: Failed to capture sample\n");
-#endif
+        PF2_DEBUG_LOG("Dropping sample: Failed to capture sample\n");
         return;
     }
 
     // Copy the sample to the ringbuffer
     if (pf2_ringbuffer_push(session->rbuf, &sample) == false) {
         // Copy failed. The sample buffer is full.
-#ifdef PF2_DEBUG
-        printf("Dropping sample: Sample buffer is full\n");
-#endif
+        PF2_DEBUG_LOG("Dropping sample: Sample buffer is full\n");
         return;
     }
 
@@ -177,7 +216,7 @@ sigprof_handler(int sig, siginfo_t *info, void *ucontext)
         (sig_end_time.tv_sec - sig_start_time.tv_sec) * 1000000000L +
         (sig_end_time.tv_nsec - sig_start_time.tv_nsec);
 
-    printf("sigprof_handler: consumed_time_ns: %lu\n", sample.consumed_time_ns);
+    PF2_DEBUG_LOG("sigprof_handler: consumed_time_ns: %lu\n", sample.consumed_time_ns);
 #endif
 }
 
@@ -220,9 +259,20 @@ rb_pf2_session_stop(VALUE self)
     session->duration_ns = end_ns - start_ns;
 
     // Disarm and delete the timer.
+#ifdef HAVE_TIMER_CREATE
     if (timer_delete(session->timer) == -1) {
         rb_raise(rb_eRuntimeError, "Failed to delete timer");
     }
+#else
+    struct itimerval zero_timer = {{0, 0}, {0, 0}};
+    int which_timer = session->configuration->time_mode == PF2_TIME_MODE_CPU_TIME
+        ? ITIMER_PROF
+        : ITIMER_REAL;
+    if (setitimer(which_timer, &zero_timer, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to stop timer");
+    }
+    global_current_session = NULL;
+#endif
 
     // Terminate the collector thread
     session->is_running = false;

data/ext/{pf2c → pf2}/session.h

@@ -3,6 +3,7 @@
 
 #include <pthread.h>
 #include <stdatomic.h>
+#include <sys/time.h>
 
 #include <ruby.h>
 
@@ -12,7 +13,11 @@
 
 struct pf2_session {
     bool is_running;
+#ifdef HAVE_TIMER_CREATE
     timer_t timer;
+#else
+    struct itimerval timer;
+#endif
     struct pf2_ringbuffer *rbuf;
     atomic_bool is_marking; // Whether garbage collection is in progress
     pthread_t *collector_thread;

data/lib/pf2/cli.rb

@@ -55,20 +55,12 @@ module Pf2
         opts.on('-o', '--output FILE', 'Output file') do |path|
           options[:output_file] = path
         end
-        opts.on('--experimental-serializer', 'Enable the experimental serializer mode') do
-          options[:experimental_serializer] = true
-        end
       end
       option_parser.parse!(argv)
 
-      if options[:experimental_serializer]
-        profile = Marshal.load(File.read(argv[0]))
-        report = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
-        report = JSON.generate(report)
-      else
-        profile = JSON.parse(File.read(argv[0]), symbolize_names: true, max_nesting: false)
-        report = JSON.generate(Pf2::Reporter::FirefoxProfiler.new(profile).emit)
-      end
+      profile = Marshal.load(File.read(argv[0]))
+      report = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
+      report = JSON.generate(report)
 
       if options[:output_file]
         File.write(options[:output_file], report)

data/lib/pf2/reporter/firefox_profiler_ser2.rb

@@ -2,6 +2,8 @@
 
 require 'json'
 
+require_relative 'stack_weaver'
+
 module Pf2
   module Reporter
     # Generates Firefox Profiler's "processed profile format"
@@ -59,7 +61,7 @@ module Pf2
           counters: [],
           threads: thread_reports,
         }
-        FirefoxProfiler.deep_camelize_keys(report)
+        FirefoxProfilerSer2.deep_camelize_keys(report)
       end
 
       class ThreadReport
@@ -72,6 +74,7 @@ module Pf2
           @stack_tree = { :stack_id => nil }
           @reverse_stack_tree = []
           @string_table = { nil => 0 }
+          @stack_weaver = StackWeaver.new(@profile)
         end
 
         def inspect
@@ -79,9 +82,6 @@ module Pf2
         end
 
         def emit
-          # TODO: weave?
-          # @thread[:stack_tree] = x
-
           # Build func table from profile[:functions]
           func_table = build_func_table
           # Build frame table from profile[:locations]
@@ -136,8 +136,10 @@ module Pf2
           }
 
           @samples.each do |sample|
-            stack = sample[:stack].reverse
-            stack_id = @stack_tree.dig(*stack, :stack_id)
+            # Weave Ruby stack and native stack
+            woven_stack = @stack_weaver.weave(sample[:stack], sample[:native_stack] || [])
+            woven_stack.reverse! # StackWeaver returns in root->leaf order, we need leaf->root
+            stack_id = @stack_tree.dig(*woven_stack, :stack_id)
 
             ret[:stack] << stack_id
             ret[:time] << sample[:elapsed_ns] / 1_000_000 # ns -> ms
@@ -165,9 +167,11 @@ module Pf2
           }
 
           @profile[:locations].each.with_index do |location, i|
+            function = @profile[:functions][location[:function_index]]
+
             ret[:address] << location[:address]
-            ret[:category] << 1
-            ret[:subcategory] << 1
+            ret[:category] << (function[:implementation] == :native ? 3 : 2)
+            ret[:subcategory] << nil
             ret[:func] << location[:function_index]
             ret[:inner_window_id] << nil
             ret[:implementation] << nil
@@ -218,15 +222,15 @@ module Pf2
           }
 
           @profile[:samples].each do |sample|
-            # Stack (Array of location indices) recorded in sample, reversed
-            # example: [1, 2, 9] (1 is the root)
-            stack = sample[:stack].reverse
+            # Weave Ruby stack and native stack
+            woven_stack = @stack_weaver.weave(sample[:stack], sample[:native_stack] || [])
+            woven_stack.reverse! # StackWeaver returns in root->leaf order, we need leaf->root
 
             # Build the stack_table Array which Firefox Profiler requires.
             # At the same time, build the stack tree for efficient traversal.
 
             current_node = @stack_tree # the stack tree root
-            stack.each do |location_index|
+            woven_stack.each do |location_index|
               if current_node[location_index].nil?
                 # The tree node is unknown. Create it.
                 new_stack_id = ret[:frame].length # The position of the new stack in the stack_table array
@@ -238,7 +242,7 @@ module Pf2
 
                 # Register the stack in the stack_table Array
                 ret[:frame] << location_index
-                ret[:category] << (function[:implementation] == :ruby ? 2 : 1)
+                ret[:category] << (function[:implementation] == :native ? 3 : 2)
                 ret[:subcategory] << nil
                 ret[:prefix] << current_node[:stack_id] # the parent's position in the stack_table array
               end

data/lib/pf2/reporter/stack_weaver.rb

@@ -2,6 +2,14 @@
 
 module Pf2
   module Reporter
+    # "Weaves" the native stack into the Ruby stack.
+    #
+    # Strategy:
+    # - Split the stack into Ruby and Native parts
+    # - Start from the root of the Native stack
+    # - Dig in to the native stack until we hit a rb_vm_exec(), which marks a call into Ruby code
+    # - Switch to Ruby stack. Keep digging until we hit a Cfunc call, then switch back to Native stack
+    # - Repeat until we consume the entire stack
     class StackWeaver
       def initialize(profile)
         @profile = profile

data/lib/pf2/reporter.rb

@@ -2,5 +2,4 @@
 
 require_relative './reporter/annotate'
 require_relative './reporter/stack_weaver'
-require_relative './reporter/firefox_profiler'
 require_relative './reporter/firefox_profiler_ser2'

data/lib/pf2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pf2
-  VERSION = '0.9.0'
+  VERSION = '1.0.0.alpha1'
 end

data/lib/pf2.rb

@@ -8,7 +8,7 @@ module Pf2
   class Error < StandardError; end
 
   def self.start(...)
-    @@session = Pf2::Session.new(...)
+    @@session = Pf2c::Session.new(...)
     @@session.start
   end