pf2 0.8.0 → 1.0.0.alpha1

data/ext/pf2/session.c

@@ -0,0 +1,394 @@
+#include <bits/time.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <time.h>
+
+#include <ruby.h>
+#include <ruby/debug.h>
+
+#include <backtrace.h>
+
+#include "backtrace_state.h"
+#include "configuration.h"
+#include "debug.h"
+#include "sample.h"
+#include "session.h"
+#include "serializer.h"
+
+#ifndef HAVE_TIMER_CREATE
+// Global session pointer for setitimer fallback
+static struct pf2_session *global_current_session = NULL;
+#endif
+
+static void *sample_collector_thread(void *arg);
+static void sigprof_handler(int sig, siginfo_t *info, void *ucontext);
+bool ensure_sample_capacity(struct pf2_session *session);
+
+VALUE
+rb_pf2_session_initialize(int argc, VALUE *argv, VALUE self)
+{
+    struct pf2_session *session;
+    TypedData_Get_Struct(self, struct pf2_session, &pf2_session_type, session);
+
+    // Create configuration from options hash
+    VALUE kwargs = Qnil;
+    rb_scan_args(argc, argv, ":", &kwargs);
+    ID kwarg_labels[] = {
+        rb_intern("interval_ms"),
+        rb_intern("time_mode")
+    };
+    VALUE *kwarg_values = NULL;
+    rb_get_kwargs(kwargs, kwarg_labels, 0, 2, kwarg_values);
+
+    session->configuration = pf2_configuration_new_from_options_hash(kwargs);
+
+    return self;
+}
+
+VALUE
+rb_pf2_session_start(VALUE self)
+{
+    struct pf2_session *session;
+    TypedData_Get_Struct(self, struct pf2_session, &pf2_session_type, session);
+
+    session->is_running = true;
+
+    // Record start time
+    clock_gettime(CLOCK_REALTIME, &session->start_time_realtime);
+    clock_gettime(CLOCK_MONOTONIC, &session->start_time);
+
+    // Spawn a collector thread which periodically wakes up and collects samples
+    if (pthread_create(session->collector_thread, NULL, sample_collector_thread, session) != 0) {
+        rb_raise(rb_eRuntimeError, "Failed to spawn sample collector thread");
+    }
+
+    // Install signal handler for SIGPROF
+    struct sigaction sa;
+    sa.sa_sigaction = sigprof_handler;
+    sigemptyset(&sa.sa_mask);
+    sigaddset(&sa.sa_mask, SIGPROF); // Mask SIGPROFs when handler is running
+    sa.sa_flags = SA_SIGINFO | SA_RESTART;
+    if (sigaction(SIGPROF, &sa, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to install SIGPROF handler");
+    }
+
+#ifndef HAVE_TIMER_CREATE
+    // Install signal handler for SIGALRM if using wall time mode with setitimer
+    if (session->configuration->time_mode != PF2_TIME_MODE_CPU_TIME) {
+        sigaddset(&sa.sa_mask, SIGALRM);
+        if (sigaction(SIGALRM, &sa, NULL) == -1) {
+            rb_raise(rb_eRuntimeError, "Failed to install SIGALRM handler");
+        }
+    }
+#endif
+
+#ifdef HAVE_TIMER_CREATE
+    // Configure a kernel timer to send SIGPROF periodically
+    struct sigevent sev;
+    sev.sigev_notify = SIGEV_SIGNAL;
+    sev.sigev_signo = SIGPROF;
+    sev.sigev_value.sival_ptr = session; // Passed as info->si_value.sival_ptr
+    if (timer_create(
+        session->configuration->time_mode == PF2_TIME_MODE_CPU_TIME
+            ? CLOCK_PROCESS_CPUTIME_ID
+            : CLOCK_MONOTONIC,
+        &sev,
+        &session->timer
+    ) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to create timer");
+    }
+    struct itimerspec its = {
+        .it_value = {
+            .tv_sec = 0,
+            .tv_nsec = session->configuration->interval_ms * 1000000,
+        },
+        .it_interval = {
+            .tv_sec = 0,
+            .tv_nsec = session->configuration->interval_ms * 1000000,
+        },
+    };
+    if (timer_settime(session->timer, 0, &its, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to start timer");
+    }
+#else
+    // Use setitimer as fallback
+    // Some platforms (e.g. macOS) do not have timer_create(3).
+    // setitimer(3) can be used as a alternative, but has limited functionality.
+    global_current_session = session;
+
+    struct itimerval itv = {
+        .it_value = {
+            .tv_sec = 0,
+            .tv_usec = session->configuration->interval_ms * 1000,
+        },
+        .it_interval = {
+            .tv_sec = 0,
+            .tv_usec = session->configuration->interval_ms * 1000,
+        },
+    };
+    int which_timer = session->configuration->time_mode == PF2_TIME_MODE_CPU_TIME
+        ? ITIMER_PROF  // CPU time (sends SIGPROF)
+        : ITIMER_REAL; // Wall time (sends SIGALRM)
+
+    if (setitimer(which_timer, &itv, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to start timer");
+    }
+#endif
+
+    return Qtrue;
+}
+
+static void *
+sample_collector_thread(void *arg)
+{
+    struct pf2_session *session = arg;
+
+    while (session->is_running == true) {
+        // Take samples from the ring buffer
+        struct pf2_sample sample;
+        while (pf2_ringbuffer_pop(session->rbuf, &sample) == true) {
+            // Ensure we have capacity before adding a new sample
+            if (!ensure_sample_capacity(session)) {
+                // Failed to expand buffer
+                PF2_DEBUG_LOG("Failed to expand sample buffer. Dropping sample\n");
+                break;
+            }
+
+            session->samples[session->samples_index++] = sample;
+        }
+
+        // Sleep for 100 ms
+        // TODO: Replace with high watermark callback
+        struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000000, }; // 10 ms
+        nanosleep(&ts, NULL);
+    }
+
+    return NULL;
+}
+
+// async-signal-safe
+static void
+sigprof_handler(int sig, siginfo_t *info, void *ucontext)
+{
+#ifdef PF2_DEBUG
+    struct timespec sig_start_time;
+    clock_gettime(CLOCK_MONOTONIC, &sig_start_time);
+#endif
+
+    struct pf2_session *session;
+#ifdef HAVE_TIMER_CREATE
+    session = info->si_value.sival_ptr;
+#else
+    session = global_current_session;
+#endif
+
+    // If garbage collection is in progress, don't collect samples.
+    if (atomic_load_explicit(&session->is_marking, memory_order_acquire)) {
+        PF2_DEBUG_LOG("Dropping sample: Garbage collection is in progress\n");
+        return;
+    }
+
+    struct pf2_sample sample;
+
+    if (pf2_sample_capture(&sample) == false) {
+        PF2_DEBUG_LOG("Dropping sample: Failed to capture sample\n");
+        return;
+    }
+
+    // Copy the sample to the ringbuffer
+    if (pf2_ringbuffer_push(session->rbuf, &sample) == false) {
+        // Copy failed. The sample buffer is full.
+        PF2_DEBUG_LOG("Dropping sample: Sample buffer is full\n");
+        return;
+    }
+
+#ifdef PF2_DEBUG
+    struct timespec sig_end_time;
+    clock_gettime(CLOCK_MONOTONIC, &sig_end_time);
+
+    // Calculate elapsed time in nanoseconds
+    sample.consumed_time_ns =
+        (sig_end_time.tv_sec - sig_start_time.tv_sec) * 1000000000L +
+        (sig_end_time.tv_nsec - sig_start_time.tv_nsec);
+
+    PF2_DEBUG_LOG("sigprof_handler: consumed_time_ns: %lu\n", sample.consumed_time_ns);
+#endif
+}
+
+// Ensures that the session's sample array has capacity for at least one more sample
+// Returns true if successful, false if memory allocation failed
+bool
+ensure_sample_capacity(struct pf2_session *session)
+{
+    // Check if we need to expand
+    if (session->samples_index < session->samples_capacity) {
+        return true;
+    }
+
+    // Calculate new size (double the current size)
+    size_t new_capacity = session->samples_capacity * 2;
+
+    // Reallocate the array
+    struct pf2_sample *new_samples = realloc(session->samples, new_capacity * sizeof(struct pf2_sample));
+    if (new_samples == NULL) {
+        return false;
+    }
+
+    session->samples = new_samples;
+    session->samples_capacity = new_capacity;
+
+    return true;
+}
+
+VALUE
+rb_pf2_session_stop(VALUE self)
+{
+    struct pf2_session *session;
+    TypedData_Get_Struct(self, struct pf2_session, &pf2_session_type, session);
+
+    // Calculate duration
+    struct timespec end_time;
+    clock_gettime(CLOCK_MONOTONIC, &end_time);
+    uint64_t start_ns = (uint64_t)session->start_time.tv_sec * 1000000000ULL + (uint64_t)session->start_time.tv_nsec;
+    uint64_t end_ns = (uint64_t)end_time.tv_sec * 1000000000ULL + (uint64_t)end_time.tv_nsec;
+    session->duration_ns = end_ns - start_ns;
+
+    // Disarm and delete the timer.
+#ifdef HAVE_TIMER_CREATE
+    if (timer_delete(session->timer) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to delete timer");
+    }
+#else
+    struct itimerval zero_timer = {{0, 0}, {0, 0}};
+    int which_timer = session->configuration->time_mode == PF2_TIME_MODE_CPU_TIME
+        ? ITIMER_PROF
+        : ITIMER_REAL;
+    if (setitimer(which_timer, &zero_timer, NULL) == -1) {
+        rb_raise(rb_eRuntimeError, "Failed to stop timer");
+    }
+    global_current_session = NULL;
+#endif
+
+    // Terminate the collector thread
+    session->is_running = false;
+    pthread_join(*session->collector_thread, NULL);
+
+    // Create serializer and serialize
+    struct pf2_ser *serializer = pf2_ser_new();
+    pf2_ser_prepare(serializer, session);
+    VALUE result = pf2_ser_to_ruby_hash(serializer);
+    pf2_ser_free(serializer);
+
+    return result;
+}
+
+VALUE
+rb_pf2_session_configuration(VALUE self)
+{
+    struct pf2_session *session;
+    TypedData_Get_Struct(self, struct pf2_session, &pf2_session_type, session);
+    return pf2_configuration_to_ruby_hash(session->configuration);
+}
+
+VALUE
+pf2_session_alloc(VALUE self)
+{
+    // Initialize state for libbacktrace
+    if (global_backtrace_state == NULL) {
+        global_backtrace_state = backtrace_create_state("pf2", 1, pf2_backtrace_print_error, NULL);
+        if (global_backtrace_state == NULL) {
+            rb_raise(rb_eRuntimeError, "Failed to initialize libbacktrace");
+        }
+    }
+
+    struct pf2_session *session = malloc(sizeof(struct pf2_session));
+    if (session == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate memory");
+    }
+
+    session->rbuf = pf2_ringbuffer_new(1000);
+    if (session->rbuf == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate memory");
+    }
+
+    atomic_store_explicit(&session->is_marking, false, memory_order_relaxed);
+    session->collector_thread = malloc(sizeof(pthread_t));
+    if (session->collector_thread == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate memory");
+    }
+
+    session->duration_ns = 0;
+
+    session->samples_index = 0;
+    session->samples_capacity = 500; // 10 seconds worth of samples at 50 Hz
+    session->samples = malloc(sizeof(struct pf2_sample) * session->samples_capacity);
+    if (session->samples == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate memory");
+    }
+
+    session->configuration = NULL;
+
+    return TypedData_Wrap_Struct(self, &pf2_session_type, session);
+}
+
+void
+pf2_session_dmark(void *sess)
+{
+    struct pf2_session *session = sess;
+
+    // Disallow sample collection during marking
+    atomic_store_explicit(&session->is_marking, true, memory_order_release);
+
+    // Iterate over all samples in the ringbuffer and mark them
+    struct pf2_ringbuffer *rbuf = session->rbuf;
+    struct pf2_sample *sample;
+    int head = atomic_load_explicit(&rbuf->head, memory_order_acquire);
+    int tail = atomic_load_explicit(&rbuf->tail, memory_order_acquire);
+    while (head != tail) {
+        sample = &rbuf->samples[head];
+        // TODO: Move this to mark function in pf2_sample
+        for (int i = 0; i < sample->depth; i++) {
+            rb_gc_mark(sample->cmes[i]);
+        }
+        head = (head + 1) % rbuf->size;
+    }
+
+    // Iterate over all samples in the samples array and mark them
+    for (size_t i = 0; i < session->samples_index; i++) {
+        sample = &session->samples[i];
+        for (int i = 0; i < sample->depth; i++) {
+            rb_gc_mark(sample->cmes[i]);
+        }
+    }
+
+    // Allow sample collection
+    atomic_store_explicit(&session->is_marking, false, memory_order_release);
+}
+
+void
+pf2_session_dfree(void *sess)
+{
+    // TODO: Ensure the uninstall process is complete before freeing the session
+    struct pf2_session *session = sess;
+    pf2_configuration_free(session->configuration);
+    pf2_ringbuffer_free(session->rbuf);
+    free(session->samples);
+    free(session->collector_thread);
+    free(session);
+}
+
+size_t
+pf2_session_dsize(const void *sess)
+{
+    const struct pf2_session *session = sess;
+    return (
+        sizeof(struct pf2_session)
+        + sizeof(struct pf2_sample) * session->samples_capacity
+        + sizeof(struct pf2_sample) * session->rbuf->size
+    );
+}

data/ext/pf2/session.h

@@ -0,0 +1,56 @@
+#ifndef PF2_SESSION_H
+#define PF2_SESSION_H
+
+#include <pthread.h>
+#include <stdatomic.h>
+#include <sys/time.h>
+
+#include <ruby.h>
+
+#include "configuration.h"
+#include "ringbuffer.h"
+#include "sample.h"
+
+struct pf2_session {
+    bool is_running;
+#ifdef HAVE_TIMER_CREATE
+    timer_t timer;
+#else
+    struct itimerval timer;
+#endif
+    struct pf2_ringbuffer *rbuf;
+    atomic_bool is_marking; // Whether garbage collection is in progress
+    pthread_t *collector_thread;
+
+    struct pf2_sample *samples; // Dynamic array of samples
+    size_t samples_index;
+    size_t samples_capacity;    // Current capacity of the samples array
+
+    struct timespec start_time_realtime;
+    struct timespec start_time; // When profiling started
+    uint64_t duration_ns; // Duration of profiling in nanoseconds
+
+    struct pf2_configuration *configuration;
+};
+
+VALUE rb_pf2_session_initialize(int argc, VALUE *argv, VALUE self);
+VALUE rb_pf2_session_start(VALUE self);
+VALUE rb_pf2_session_stop(VALUE self);
+VALUE rb_pf2_session_configuration(VALUE self);
+VALUE pf2_session_alloc(VALUE self);
+void pf2_session_dmark(void *sess);
+void pf2_session_dfree(void *sess);
+size_t pf2_session_dsize(const void *sess);
+
+static const rb_data_type_t pf2_session_type = {
+    .wrap_struct_name = "Pf2c::Session",
+    .function = {
+        .dmark = pf2_session_dmark,
+        .dfree = pf2_session_dfree,
+        .dsize = pf2_session_dsize,
+    },
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+#endif // PF2_SESSION_H

data/lib/pf2/cli.rb

@@ -19,6 +19,8 @@ module Pf2
       case subcommand
       when 'report'
         subcommand_report(argv)
+      when 'annotate'
+        subcommand_annotate(argv)
       when 'serve'
         subcommand_serve(argv)
       when 'version'
@@ -53,20 +55,12 @@ module Pf2
         opts.on('-o', '--output FILE', 'Output file') do |path|
           options[:output_file] = path
         end
-        opts.on('--experimental-serializer', 'Enable the experimental serializer mode') do
-          options[:experimental_serializer] = true
-        end
       end
       option_parser.parse!(argv)
 
-      if options[:experimental_serializer]
-        profile = Marshal.load(File.read(argv[0]))
-        report = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
-        report = JSON.generate(report)
-      else
-        profile = JSON.parse(File.read(argv[0]), symbolize_names: true, max_nesting: false)
-        report = JSON.generate(Pf2::Reporter::FirefoxProfiler.new(profile).emit)
-      end
+      profile = Marshal.load(File.read(argv[0]))
+      report = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
+      report = JSON.generate(report)
 
       if options[:output_file]
         File.write(options[:output_file], report)
@@ -77,6 +71,26 @@ module Pf2
       return 0
     end
 
+    def subcommand_annotate(argv)
+      options = {}
+      option_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: pf2 report [options] COMMAND"
+        opts.on('-h', '--help', 'Prints this help') do
+          puts opts
+          return 0
+        end
+        opts.on('-d', '--source-directory DIR', 'Path to the source directory') do |dir|
+          options[:source_directory] = dir
+        end
+      end
+      option_parser.parse!(argv)
+
+      profile = Marshal.load(File.binread(argv[0]))
+      Pf2::Reporter::Annotate.new(profile, options[:source_directory] || '.').annotate
+
+      return 0
+    end
+
     def subcommand_serve(argv)
       options = {}
       option_parser = OptionParser.new do |opts|

data/lib/pf2/reporter/annotate.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Pf2
+  module Reporter
+    class Annotate
+      HitCount = Struct.new(:self, :total, keyword_init: false)
+      SourceCodeHits = Struct.new(:path, :line_count, keyword_init: false)
+
+      # @param profile [Hash]
+      # @param source_directory [String]
+      def initialize(profile, source_directory)
+        @profile = profile
+        @source_directory = source_directory
+      end
+
+      def annotate
+        tallied = tally_by_source_code_line(@profile)
+
+        # Print the source code with hit counts
+        tallied.each do |path, source_code_hits|
+          expanded_path = File.expand_path(path, @source_directory)
+          if !File.exist?(expanded_path)
+            if ignorable_path?(path)
+              puts "Ignoring file: #{path}"
+            else
+              puts "File not found: #{path}"
+            end
+            puts ""
+            puts ""
+            next
+          end
+          source_file = File.open(expanded_path, "r")
+
+          puts expanded_path
+          puts ""
+
+          # Print in tabular format
+
+          # Header row
+          puts "  ttl  self │"
+
+          source_file.each_line.with_index(1) do |line, lineno|
+            hits = source_code_hits.line_count[lineno]
+
+            if !hits.nil?
+              # If any samples are captured for this line
+              puts "%5d %5d │ %s" % [hits.total, hits.self, line.chomp]
+            else
+              puts "%5s %5s │ %s" % ["", "", line.chomp]
+            end
+          end
+
+          puts ""
+          puts ""
+        ensure
+          source_file.close if source_file
+        end
+      end
+
+      # @return [Array<SourceCodeHits>]
+      private def tally_by_source_code_line(profile)
+        # Iterate over all samples and tally self hits and total hits by location
+        hits_per_location = {}
+        @profile[:samples].each do |sample|
+          # Record a total hit for all locations in the stack
+          sample[:stack].each do |location_id|
+            hits_per_location[location_id] ||= HitCount.new(0, 0)
+            hits_per_location[location_id].total += 1
+          end
+
+          # Record a self hit for the topmost stack frame, which is the first element in the array
+          topmost_location_id = sample[:stack][0]
+          hits_per_location[topmost_location_id].self += 1
+        end
+
+        # Associate a filename and lineno for each location
+        hits_per_file = {}
+        hits_per_location.each do |location_id, hits|
+          location = @profile[:locations][location_id]
+          function = @profile[:functions][location[:function_index]]
+
+          filename = function[:filename]
+          # Some locations simply cannot be associated to a specific file.
+          # We just ignore them.
+          next if filename.nil?
+          lineno = location[:lineno]
+
+          hits_per_file[filename] ||= SourceCodeHits.new(filename, {})
+          hits_per_file[filename].line_count[lineno] = hits
+        end
+
+        hits_per_file
+      end
+
+      private def ignorable_path?(path)
+        return true if path.start_with?("<internal:")
+        false
+      end
+    end
+  end
+end

data/lib/pf2/reporter/firefox_profiler_ser2.rb

@@ -2,6 +2,8 @@
 
 require 'json'
 
+require_relative 'stack_weaver'
+
 module Pf2
   module Reporter
     # Generates Firefox Profiler's "processed profile format"
@@ -59,7 +61,7 @@ module Pf2
           counters: [],
           threads: thread_reports,
         }
-        FirefoxProfiler.deep_camelize_keys(report)
+        FirefoxProfilerSer2.deep_camelize_keys(report)
       end
 
       class ThreadReport
@@ -72,6 +74,7 @@ module Pf2
           @stack_tree = { :stack_id => nil }
           @reverse_stack_tree = []
           @string_table = { nil => 0 }
+          @stack_weaver = StackWeaver.new(@profile)
         end
 
         def inspect
@@ -79,9 +82,6 @@ module Pf2
         end
 
         def emit
-          # TODO: weave?
-          # @thread[:stack_tree] = x
-
           # Build func table from profile[:functions]
           func_table = build_func_table
           # Build frame table from profile[:locations]
@@ -136,8 +136,10 @@ module Pf2
           }
 
           @samples.each do |sample|
-            stack = sample[:stack].reverse
-            stack_id = @stack_tree.dig(*stack, :stack_id)
+            # Weave Ruby stack and native stack
+            woven_stack = @stack_weaver.weave(sample[:stack], sample[:native_stack] || [])
+            woven_stack.reverse! # StackWeaver returns in root->leaf order, we need leaf->root
+            stack_id = @stack_tree.dig(*woven_stack, :stack_id)
 
             ret[:stack] << stack_id
             ret[:time] << sample[:elapsed_ns] / 1_000_000 # ns -> ms
@@ -165,9 +167,11 @@ module Pf2
           }
 
           @profile[:locations].each.with_index do |location, i|
+            function = @profile[:functions][location[:function_index]]
+
             ret[:address] << location[:address]
-            ret[:category] << 1
-            ret[:subcategory] << 1
+            ret[:category] << (function[:implementation] == :native ? 3 : 2)
+            ret[:subcategory] << nil
             ret[:func] << location[:function_index]
             ret[:inner_window_id] << nil
             ret[:implementation] << nil
@@ -218,15 +222,15 @@ module Pf2
           }
 
           @profile[:samples].each do |sample|
-            # Stack (Array of location indices) recorded in sample, reversed
-            # example: [1, 2, 9] (1 is the root)
-            stack = sample[:stack].reverse
+            # Weave Ruby stack and native stack
+            woven_stack = @stack_weaver.weave(sample[:stack], sample[:native_stack] || [])
+            woven_stack.reverse! # StackWeaver returns in root->leaf order, we need leaf->root
 
             # Build the stack_table Array which Firefox Profiler requires.
             # At the same time, build the stack tree for efficient traversal.
 
             current_node = @stack_tree # the stack tree root
-            stack.each do |location_index|
+            woven_stack.each do |location_index|
               if current_node[location_index].nil?
                 # The tree node is unknown. Create it.
                 new_stack_id = ret[:frame].length # The position of the new stack in the stack_table array
@@ -238,7 +242,7 @@ module Pf2
 
                 # Register the stack in the stack_table Array
                 ret[:frame] << location_index
-                ret[:category] << (function[:implementation] == :ruby ? 2 : 1)
+                ret[:category] << (function[:implementation] == :native ? 3 : 2)
                 ret[:subcategory] << nil
                 ret[:prefix] << current_node[:stack_id] # the parent's position in the stack_table array
               end