pf2 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66bf1eb8d6457621a04819a8d100bdb109bf1fd7b4820eb1423a85dcfb91799c
-  data.tar.gz: b1046c04ae8a2a0630d3bd2ed86c4dc645a9ca42c8038bfaaee0be4c71dbec78
+  metadata.gz: 1aa1dd1d3caa1dd9327382e7de0a44b2c0ff4555a9dff8fdc7244f61c3a13405
+  data.tar.gz: c50895ea4d8285cc9bb223d575cbec7042aacd119a1390b836b818232ef7e214
 SHA512:
-  metadata.gz: 5e8854a44c477c42dc9baa0a5b2cd95e19cd09a3c41fad0e950e7de6bd88ff90bf6a3d8e7b61c43c966976211047d5ecb6da11fbe5cba6460ab456186856e2b8
-  data.tar.gz: 8d577d2502f38dab61d3ab130f2555b314667c414ec13b2829f2e7fdf0ddf1f114310d560806685145e5ce65f7f24cc50f834f9fc6eb2b1e0d7763b8ebe69d5f
+  metadata.gz: 1b5db496a58016e11203334b71e80e7a4c0d7d0dce6b2cb288884da4aecd8820d8915fd0b9f4ebd546802c314d621f69da23c474d254bff17ec3931cb8f7604b
+  data.tar.gz: 5e3a8fcf25be2dbf28cf8c36e03839cbeed13aa8d944e43991672a39f06fe16f6eae07aab5cd282c3237171bfff94b39a8abb176153a091452c3177266eb1364

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.6.0] - 2024-07-15
+
+### Changed
+
+- The default sampling interval is now 9 ms (was originally 49 ms).
+  - It is intentional that the default is not 10 (or 50) ms - this is to avoid lockstep sampling.
+- BREAKING: `Pf2::Reporter` has been moved to `Pf2::Reporter::FirefoxProfiler`.
+  - This is to make space for other planned reporters.
+
+
 ## [0.5.2] - 2024-07-13
 
 ### Fixed

data/README.md

@@ -67,7 +67,7 @@ Pf2 accepts the following configuration keys:
 
 ```rb
 Pf2.start(
-  interval_ms: 49,        # Integer: The sampling interval in milliseconds (default: 49)
+  interval_ms: 9,        # Integer: The sampling interval in milliseconds (default: 9)
   time_mode: :cpu,        # `:cpu` or `:wall`: The sampling timer's mode
                           # (default: `:cpu` for SignalScheduler, `:wall` for TimerThreadScheduler)
   threads: [th1, th2],    # `Array<Thread>` | `:all`: A list of Ruby Threads to be tracked.

data/ext/pf2/src/lib.rs

@@ -10,6 +10,7 @@ mod profile_serializer;
 mod ringbuffer;
 mod sample;
 mod scheduler;
+mod serialization;
 mod session;
 #[cfg(target_os = "linux")]
 mod signal_scheduler;

data/ext/pf2/src/profile.rs

@@ -1,4 +1,4 @@
-use std::time::Instant;
+use std::time::{Instant, SystemTime};
 use std::{collections::HashSet, ptr::null_mut};
 
 use rb_sys::*;
@@ -15,7 +15,9 @@ const DEFAULT_RINGBUFFER_CAPACITY: usize = 320;
 
 #[derive(Debug)]
 pub struct Profile {
-    pub start_timestamp: Instant,
+    pub start_timestamp: SystemTime,
+    pub start_instant: Instant,
+    pub end_instant: Option<Instant>,
     pub samples: Vec<Sample>,
     pub temporary_sample_buffer: Ringbuffer,
     pub backtrace_state: BacktraceState,
@@ -35,7 +37,9 @@ impl Profile {
         };
 
         Self {
-            start_timestamp: Instant::now(),
+            start_timestamp: SystemTime::now(),
+            start_instant: Instant::now(),
+            end_instant: None,
             samples: vec![],
             temporary_sample_buffer: Ringbuffer::new(DEFAULT_RINGBUFFER_CAPACITY),
             backtrace_state,

data/ext/pf2/src/profile_serializer.rs

@@ -221,7 +221,7 @@ impl ProfileSerializer {
 
                     if merged_stack.is_empty() {
                         // This is the leaf node, record a Sample
-                        let elapsed_ns = (sample.timestamp - profile.start_timestamp).as_nanos();
+                        let elapsed_ns = (sample.timestamp - profile.start_instant).as_nanos();
                         thread_serializer.samples.push(ProfileSample {
                             elapsed_ns,
                             stack_tree_id: stack_tree.node_id,

data/ext/pf2/src/serialization/profile.rs

@@ -0,0 +1,46 @@
+#[derive(Clone, Deserialize, Serialize)]
+pub struct Profile {
+    pub samples: Vec<Sample>,
+    pub locations: Vec<Location>,
+    pub functions: Vec<Function>,
+    pub start_timestamp_ns: u128,
+    pub duration_ns: u128,
+}
+
+pub type LocationIndex = usize;
+pub type FunctionIndex = usize;
+
+/// Sample
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Sample {
+    /// The stack leading to this sample.
+    /// The leaf node will be stored at `stack[0]`.
+    pub stack: Vec<LocationIndex>,
+    pub ruby_thread_id: Option<u64>,
+}
+
+/// Location represents a location (line) in the source code when a sample was captured.
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
+pub struct Location {
+    pub function_index: FunctionIndex,
+    pub lineno: i32,
+    pub address: Option<usize>,
+}
+
+/// Function represents a Ruby method or a C function in the profile.
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
+pub struct Function {
+    pub implementation: FunctionImplementation,
+    pub name: Option<String>, // unique key
+    pub filename: Option<String>,
+    /// The first line number in the method/function definition.
+    /// For the actual location (line) which was hit during sample capture, refer to `Location.lineno`.
+    pub start_lineno: Option<i32>,
+    pub start_address: Option<usize>,
+}
+
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
+pub enum FunctionImplementation {
+    Ruby,
+    C,
+}

data/ext/pf2/src/serialization/serializer.rs

@@ -0,0 +1,146 @@
+use std::ffi::CStr;
+
+use rb_sys::*;
+
+use super::profile::{Function, FunctionImplementation, Location, LocationIndex, Profile, Sample};
+use crate::util::RTEST;
+
+pub struct ProfileSerializer2 {
+    profile: Profile,
+}
+
+impl ProfileSerializer2 {
+    pub fn new() -> ProfileSerializer2 {
+        ProfileSerializer2 {
+            profile: Profile {
+                start_timestamp_ns: 0,
+                duration_ns: 0,
+                samples: vec![],
+                locations: vec![],
+                functions: vec![],
+            },
+        }
+    }
+
+    pub fn serialize(&mut self, source: &crate::profile::Profile) -> String {
+        // Fill in meta fields
+        self.profile.start_timestamp_ns = source
+            .start_timestamp
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_nanos();
+        self.profile.duration_ns = source
+            .end_instant
+            .unwrap()
+            .duration_since(source.start_instant)
+            .as_nanos();
+
+        // Create a Sample for each sample collected
+        for sample in source.samples.iter() {
+            let mut stack: Vec<LocationIndex> = vec![];
+
+            // Iterate over the Ruby stack
+            let ruby_stack_depth = sample.line_count;
+            for i in 0..ruby_stack_depth {
+                let frame: VALUE = sample.frames[i as usize];
+                let lineno: i32 = sample.linenos[i as usize];
+
+                // Get the Location corresponding to the frame.
+                let location_index = self.process_ruby_frame(frame, lineno);
+
+                stack.push(location_index);
+            }
+
+            self.profile.samples.push(Sample {
+                stack,
+                ruby_thread_id: Some(sample.ruby_thread),
+            });
+        }
+
+        serde_json::to_string(&self.profile).unwrap()
+    }
+
+    /// Process a collected Ruby frame.
+    /// Calling this method will modify `self.profile` in place.
+    ///
+    /// Returns the index of the location in `locations`.
+    fn process_ruby_frame(&mut self, frame: VALUE, lineno: i32) -> LocationIndex {
+        // Build a Function corresponding to the frame, and get the index in `functions`
+        let function = Self::extract_function_from_frame(frame);
+        let function_index = match self
+            .profile
+            .functions
+            .iter_mut()
+            .position(|f| *f == function)
+        {
+            Some(index) => index,
+            None => {
+                self.profile.functions.push(function);
+                self.profile.functions.len() - 1
+            }
+        };
+
+        // Build a Location based on (1) the Function and (2) the actual line hit during sampling.
+        let location = Location {
+            function_index,
+            lineno,
+            address: None,
+        };
+        // Get the index of the location in `locations`
+        match self
+            .profile
+            .locations
+            .iter_mut()
+            .position(|l| *l == location)
+        {
+            Some(index) => index,
+            None => {
+                self.profile.locations.push(location);
+                self.profile.locations.len() - 1
+            }
+        }
+    }
+
+    fn extract_function_from_frame(frame: VALUE) -> Function {
+        unsafe {
+            let mut frame_full_label: VALUE = rb_profile_frame_full_label(frame);
+            let frame_full_label: Option<String> = if RTEST(frame_full_label) {
+                Some(
+                    CStr::from_ptr(rb_string_value_cstr(&mut frame_full_label))
+                        .to_str()
+                        .unwrap()
+                        .to_owned(),
+                )
+            } else {
+                None
+            };
+
+            let mut frame_path: VALUE = rb_profile_frame_path(frame);
+            let frame_path: Option<String> = if RTEST(frame_path) {
+                Some(
+                    CStr::from_ptr(rb_string_value_cstr(&mut frame_path))
+                        .to_str()
+                        .unwrap()
+                        .to_owned(),
+                )
+            } else {
+                None
+            };
+
+            let frame_first_lineno: VALUE = rb_profile_frame_first_lineno(frame);
+            let frame_first_lineno: Option<i32> = if RTEST(frame_first_lineno) {
+                Some(rb_num2int(frame_first_lineno).try_into().unwrap())
+            } else {
+                None
+            };
+
+            Function {
+                implementation: FunctionImplementation::Ruby,
+                name: frame_full_label,
+                filename: frame_path,
+                start_lineno: frame_first_lineno,
+                start_address: None,
+            }
+        }
+    }
+}

data/ext/pf2/src/serialization.rs

@@ -0,0 +1,2 @@
+pub mod profile;
+pub mod serializer;

data/ext/pf2/src/session/configuration.rs

@@ -15,7 +15,7 @@ pub const DEFAULT_SCHEDULER: Scheduler = Scheduler::TimerThread;
 #[cfg(not(target_os = "linux"))]
 pub const DEFAULT_TIME_MODE: TimeMode = TimeMode::WallTime;
 
-pub const DEFAULT_INTERVAL: Duration = Duration::from_millis(49);
+pub const DEFAULT_INTERVAL: Duration = Duration::from_millis(9);
 
 #[derive(Clone, Debug)]
 pub struct Configuration {
@@ -23,6 +23,7 @@ pub struct Configuration {
     pub interval: Duration,
     pub time_mode: TimeMode,
     pub target_ruby_threads: Threads,
+    pub use_experimental_serializer: bool,
 }
 
 #[derive(Clone, Debug, PartialEq)]

data/ext/pf2/src/session.rs

@@ -38,7 +38,7 @@ impl Session {
         unsafe {
             rb_scan_args(argc, argv, cstr!(":"), &kwargs);
         };
-        let mut kwargs_values: [VALUE; 4] = [Qnil.into(); 4];
+        let mut kwargs_values: [VALUE; 5] = [Qnil.into(); 5];
         unsafe {
             rb_get_kwargs(
                 kwargs,
@@ -47,10 +47,11 @@ impl Session {
                     rb_intern(cstr!("threads")),
                     rb_intern(cstr!("time_mode")),
                     rb_intern(cstr!("scheduler")),
+                    rb_intern(cstr!("use_experimental_serializer")),
                 ]
                 .as_mut_ptr(),
                 0,
-                4,
+                5,
                 kwargs_values.as_mut_ptr(),
             );
         };
@@ -59,12 +60,15 @@ impl Session {
         let threads = Self::parse_option_threads(kwargs_values[1]);
         let time_mode = Self::parse_option_time_mode(kwargs_values[2]);
         let scheduler = Self::parse_option_scheduler(kwargs_values[3]);
+        let use_experimental_serializer =
+            Self::parse_option_use_experimental_serializer(kwargs_values[4]);
 
         let configuration = Configuration {
             scheduler,
             interval,
             target_ruby_threads: threads.clone(),
             time_mode,
+            use_experimental_serializer,
         };
 
         match configuration.validate() {
@@ -127,10 +131,10 @@ impl Session {
         let interval_ms = unsafe { rb_num2long(value) };
         Duration::from_millis(interval_ms.try_into().unwrap_or_else(|_| {
             eprintln!(
-                "[Pf2] Warning: Specified interval ({}) is not valid. Using default value (49ms).",
+                "[Pf2] Warning: Specified interval ({}) is not valid. Using default value (9ms).",
                 interval_ms
             );
-            49
+            9
         }))
     }
 
@@ -207,6 +211,13 @@ impl Session {
         scheduler
     }
 
+    fn parse_option_use_experimental_serializer(value: VALUE) -> bool {
+        if value == Qundef as VALUE {
+            return false;
+        }
+        RTEST(value)
+    }
+
     pub fn start(&mut self) -> VALUE {
         self.running.store(true, Ordering::Relaxed);
         self.start_profile_buffer_flusher_thread();

data/ext/pf2/src/signal_scheduler.rs

@@ -5,6 +5,7 @@ use crate::profile_serializer::ProfileSerializer;
 use crate::ruby_internal_apis::rb_thread_getcpuclockid;
 use crate::sample::Sample;
 use crate::scheduler::Scheduler;
+use crate::serialization::serializer::ProfileSerializer2;
 use crate::session::configuration::{self, Configuration};
 
 use core::panic;
@@ -46,6 +47,7 @@ impl Scheduler for SignalScheduler {
         match self.profile.try_write() {
             Ok(mut profile) => {
                 profile.flush_temporary_sample_buffer();
+                profile.end_instant = Some(std::time::Instant::now());
             }
             Err(_) => {
                 println!("[pf2 ERROR] stop: Failed to acquire profile lock.");
@@ -56,7 +58,11 @@ impl Scheduler for SignalScheduler {
         let profile = self.profile.try_read().unwrap();
         log::debug!("Number of samples: {}", profile.samples.len());
 
-        let serialized = ProfileSerializer::serialize(&profile);
+        let serialized = if self.configuration.use_experimental_serializer {
+            ProfileSerializer2::new().serialize(&profile)
+        } else {
+            ProfileSerializer::serialize(&profile)
+        };
         let serialized = CString::new(serialized).unwrap();
         unsafe { rb_str_new_cstr(serialized.as_ptr()) }
     }

data/ext/pf2/src/timer_thread_scheduler.rs

@@ -12,6 +12,7 @@ use crate::profile::Profile;
 use crate::profile_serializer::ProfileSerializer;
 use crate::sample::Sample;
 use crate::scheduler::Scheduler;
+use crate::serialization::serializer::ProfileSerializer2;
 use crate::session::configuration::{self, Configuration};
 use crate::util::*;
 
@@ -61,6 +62,7 @@ impl Scheduler for TimerThreadScheduler {
         match self.profile.try_write() {
             Ok(mut profile) => {
                 profile.flush_temporary_sample_buffer();
+                profile.end_instant = Some(std::time::Instant::now());
             }
             Err(_) => {
                 println!("[pf2 ERROR] stop: Failed to acquire profile lock.");
@@ -71,7 +73,11 @@ impl Scheduler for TimerThreadScheduler {
         let profile = self.profile.try_read().unwrap();
         log::debug!("Number of samples: {}", profile.samples.len());
 
-        let serialized = ProfileSerializer::serialize(&profile);
+        let serialized = if self.configuration.use_experimental_serializer {
+            ProfileSerializer2::new().serialize(&profile)
+        } else {
+            ProfileSerializer::serialize(&profile)
+        };
         let serialized = CString::new(serialized).unwrap();
         unsafe { rb_str_new_cstr(serialized.as_ptr()) }
     }

data/lib/pf2/cli.rb

@@ -55,7 +55,7 @@ module Pf2
       option_parser.parse!(argv)
 
       profile = JSON.parse(File.read(argv[0]), symbolize_names: true, max_nesting: false)
-      report = JSON.generate(Pf2::Reporter.new(profile).emit)
+      report = JSON.generate(Pf2::Reporter::FirefoxProfiler.new(profile).emit)
 
       if options[:output_file]
         File.write(options[:output_file], report)

data/lib/pf2/reporter/firefox_profiler.rb

@@ -0,0 +1,395 @@
+require 'json'
+
+module Pf2
+  module Reporter
+    # Generates Firefox Profiler's "processed profile format"
+    # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/processed-profile-format.md
+    class FirefoxProfiler
+      def initialize(profile)
+        @profile = FirefoxProfiler.deep_intize_keys(profile)
+      end
+
+      def inspect
+        "" # TODO: provide something better
+      end
+
+      def emit
+        report = {
+          meta: {
+            interval: 10, # ms; TODO: replace with actual interval
+            start_time: 0,
+            process_type: 0,
+            product: 'ruby',
+            stackwalk: 0,
+            version: 28,
+            preprocessed_profile_version: 47,
+            symbolicated: true,
+            categories: [
+              {
+                name: "Logs",
+                color: "grey",
+                subcategories: ["Unused"],
+              },
+              {
+                name: "Ruby",
+                color: "red",
+                subcategories: ["Code"],
+              },
+              {
+                name: "Native",
+                color: "blue",
+                subcategories: ["Code"],
+              },
+              {
+                name: "Native",
+                color: "lightblue",
+                subcategories: ["Code"],
+              },
+            ],
+            marker_schema: [],
+          },
+          libs: [],
+          counters: [],
+          threads: @profile[:threads].values.map {|th| ThreadReport.new(th).emit }
+        }
+        FirefoxProfiler.deep_camelize_keys(report)
+      end
+
+      class ThreadReport
+        def initialize(thread)
+          @thread = thread
+
+          # Populated in other methods
+          @func_id_map = {}
+          @frame_id_map = {}
+          @stack_tree_id_map = {}
+
+          @string_table = {}
+        end
+
+        def inspect
+          "" # TODO: provide something better
+        end
+
+        def emit
+          x = weave_native_stack(@thread[:stack_tree])
+          @thread[:stack_tree] = x
+          func_table = build_func_table
+          frame_table = build_frame_table
+          stack_table = build_stack_table(func_table, frame_table)
+          samples = build_samples
+
+          string_table = build_string_table
+
+          {
+            process_type: 'default',
+            process_name: 'ruby',
+            process_startup_time: 0,
+            process_shutdown_time: nil,
+            register_time: 0,
+            unregister_time: nil,
+            paused_ranges: [],
+            name: "Thread (tid: #{@thread[:thread_id]})",
+            is_main_thread: true,
+            is_js_tracer: true,
+            # FIXME: We can fill the correct PID only after we correctly fill is_main_thread
+            # (only one thread could be marked as is_main_thread in a single process)
+            pid: @thread[:thread_id],
+            tid: @thread[:thread_id],
+            samples: samples,
+            markers: markers,
+            stack_table: stack_table,
+            frame_table: frame_table,
+            string_array: build_string_table,
+            func_table: func_table,
+            resource_table: {
+              lib: [],
+              name: [],
+              host: [],
+              type: [],
+              length: 0,
+            },
+            native_symbols: [],
+          }
+        end
+
+        def build_samples
+          ret = {
+            event_delay: [],
+            stack: [],
+            time: [],
+            duration: [],
+            # weight: nil,
+            # weight_type: 'samples',
+          }
+
+          @thread[:samples].each do |sample|
+            ret[:stack] << @stack_tree_id_map[sample[:stack_tree_id]]
+            ret[:time] << sample[:elapsed_ns] / 1000000 # ns -> ms
+            ret[:duration] << 1
+            ret[:event_delay] << 0
+          end
+
+          ret[:length] = ret[:stack].length
+          ret
+        end
+
+        def build_frame_table
+          ret = {
+            address: [],
+            category: [],
+            subcategory: [],
+            func: [],
+            inner_window_id: [],
+            implementation: [],
+            line: [],
+            column: [],
+            optimizations: [],
+            inline_depth: [],
+            native_symbol: [],
+          }
+
+          @thread[:frames].each.with_index do |(id, frame), i|
+            ret[:address] << frame[:address].to_s
+            ret[:category] << 1
+            ret[:subcategory] << 1
+            ret[:func] << i # TODO
+            ret[:inner_window_id] << nil
+            ret[:implementation] << nil
+            ret[:line] << frame[:callsite_lineno]
+            ret[:column] << nil
+            ret[:optimizations] << nil
+            ret[:inline_depth] << 0
+            ret[:native_symbol] << nil
+
+            @frame_id_map[id] = i
+          end
+
+          ret[:length] = ret[:address].length
+          ret
+        end
+
+        def build_func_table
+          ret = {
+            name: [],
+            is_js: [],
+            relevant_for_js: [],
+            resource: [],
+            file_name: [],
+            line_number: [],
+            column_number: [],
+          }
+
+          @thread[:frames].each.with_index do |(id, frame), i|
+            native = (frame[:entry_type] == 'Native')
+            label = "#{native ? "Native: " : ""}#{frame[:full_label]}"
+            ret[:name] << string_id(label)
+            ret[:is_js] << !native
+            ret[:relevant_for_js] << false
+            ret[:resource] << -1
+            ret[:file_name] << string_id(frame[:file_name])
+            ret[:line_number] << frame[:function_first_lineno]
+            ret[:column_number] << nil
+
+            @func_id_map[id] = i
+          end
+
+          ret[:length] = ret[:name].length
+          ret
+        end
+
+        # "Weave" the native stack into the Ruby stack.
+        #
+        # Strategy:
+        # - Split the stack into Ruby and Native parts
+        # - Start from the root of the Native stack
+        # - Dig in to the native stack until we hit a rb_vm_exec(), which marks a call into Ruby code
+        # - Switch to Ruby stack. Keep digging until we hit a Cfunc call, then switch back to Native stack
+        # - Repeat until we consume the entire stack
+        def weave_native_stack(stack_tree)
+          collected_paths = []
+          tree_to_array_of_paths(stack_tree, @thread[:frames], [], collected_paths)
+          collected_paths = collected_paths.map do |path|
+            next if path.size == 0
+
+            new_path = []
+            new_path << path.shift # root
+
+            # Split the stack into Ruby and Native parts
+            native_path, ruby_path = path.partition do |frame|
+              frame_id = frame[:frame_id]
+              @thread[:frames][frame_id][:entry_type] == 'Native'
+            end
+
+            mode = :native
+
+            loop do
+              break if ruby_path.size == 0 && native_path.size == 0
+
+              case mode
+              when :ruby
+                if ruby_path.size == 0
+                  mode = :native
+                  next
+                end
+
+                next_node = ruby_path[0]
+                new_path << ruby_path.shift
+                next_node_frame = @thread[:frames][next_node[:frame_id]]
+                if native_path.size > 0
+                  # Search the remainder of the native stack for the same address
+                  # Note: This isn't a very efficient way for the job... but it still works
+                  ruby_addr = next_node_frame[:address]
+                  native_path[0..].each do |native_node|
+                    native_addr = @thread[:frames][native_node[:frame_id]][:address]
+                    if ruby_addr && native_addr && ruby_addr == native_addr
+                      # A match has been found. Switch to native mode
+                      mode = :native
+                      break
+                    end
+                  end
+                end
+              when :native
+                if native_path.size == 0
+                  mode = :ruby
+                  next
+                end
+
+                # Dig until we meet a rb_vm_exec
+                next_node = native_path[0]
+                new_path << native_path.shift
+                if @thread[:frames][next_node[:frame_id]][:full_label] =~ /vm_exec_core/ # VM_EXEC in vm_exec.h
+                  mode = :ruby
+                end
+              end
+            end
+
+            new_path
+          end
+
+          # reconstruct stack_tree
+          new_stack_tree = array_of_paths_to_tree(collected_paths)
+          new_stack_tree
+        end
+
+        def tree_to_array_of_paths(stack_tree, frames, path, collected_paths)
+          new_path = path + [{ frame_id: stack_tree[:frame_id], node_id: stack_tree[:node_id] }]
+          if stack_tree[:children].empty?
+            collected_paths << new_path
+          else
+            stack_tree[:children].each do |frame_id, child|
+              tree_to_array_of_paths(child, frames, new_path, collected_paths)
+            end
+          end
+        end
+
+        def array_of_paths_to_tree(paths)
+          new_stack_tree = { children: {}, node_id: 0, frame_id: 0 }
+          paths.each do |path|
+            current = new_stack_tree
+            path[1..].each do |frame|
+              frame_id = frame[:frame_id]
+              node_id = frame[:node_id]
+              current[:children][frame_id] ||= { children: {}, node_id: node_id, frame_id: frame_id }
+              current = current[:children][frame_id]
+            end
+          end
+          new_stack_tree
+        end
+
+        def build_stack_table(func_table, frame_table)
+          ret = {
+            frame: [],
+            category: [],
+            subcategory: [],
+            prefix: [],
+          }
+
+          queue = []
+
+          @thread[:stack_tree][:children].each {|_, c| queue << [nil, c] }
+
+          loop do
+            break if queue.size == 0
+
+            prefix, node = queue.shift
+            ret[:frame] << @frame_id_map[node[:frame_id]]
+            ret[:category] << (build_string_table[func_table[:name][frame_table[:func][@frame_id_map[node[:frame_id]]]]].start_with?('Native:') ? 2 : 1)
+            ret[:subcategory] << nil
+            ret[:prefix] << prefix
+
+            # The index of this frame - children can refer to this frame using this index as prefix
+            frame_index = ret[:frame].length - 1
+            @stack_tree_id_map[node[:node_id]] = frame_index
+
+            # Enqueue children nodes
+            node[:children].each {|_, c| queue << [frame_index, c] }
+          end
+
+          ret[:length] = ret[:frame].length
+          ret
+        end
+
+        def build_string_table
+          @string_table.sort_by {|_, v| v}.map {|s| s[0] }
+        end
+
+        def string_id(str)
+          return @string_table[str] if @string_table.has_key?(str)
+          @string_table[str] = @string_table.length
+          @string_table[str]
+        end
+
+        def markers
+          {
+            data: [],
+            name: [],
+            time: [],
+            start_time: [],
+            end_time: [],
+            phase: [],
+            category: [],
+            length: 0
+          }
+        end
+      end
+
+      # Util functions
+      class << self
+        def snake_to_camel(s)
+          return "isJS" if s == "is_js"
+          return "relevantForJS" if s == "relevant_for_js"
+          return "innerWindowID" if s == "inner_window_id"
+          s.split('_').inject([]) {|buffer, p| buffer.push(buffer.size == 0 ? p : p.capitalize) }.join
+        end
+
+        def deep_transform_keys(value, &block)
+          case value
+          when Array
+            value.map {|v| deep_transform_keys(v, &block) }
+          when Hash
+            Hash[value.map {|k, v| [yield(k), deep_transform_keys(v, &block)] }]
+          else
+            value
+          end
+        end
+
+        def deep_camelize_keys(value)
+          deep_transform_keys(value) do |key|
+            snake_to_camel(key.to_s).to_sym
+          end
+        end
+
+        def deep_intize_keys(value)
+          deep_transform_keys(value) do |key|
+            if key.to_s.to_i.to_s == key.to_s
+              key.to_s.to_i
+            else
+              key
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pf2/reporter.rb

@@ -1,393 +1,5 @@
-require 'json'
+require_relative './reporter/firefox_profiler'
 
 module Pf2
-  # Generates Firefox Profiler's "processed profile format"
-  # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/processed-profile-format.md
-  class Reporter
-    def initialize(profile)
-      @profile = Reporter.deep_intize_keys(profile)
-    end
-
-    def inspect
-      "" # TODO: provide something better
-    end
-
-    def emit
-      report = {
-        meta: {
-          interval: 10, # ms; TODO: replace with actual interval
-          start_time: 0,
-          process_type: 0,
-          product: 'ruby',
-          stackwalk: 0,
-          version: 28,
-          preprocessed_profile_version: 47,
-          symbolicated: true,
-          categories: [
-            {
-              name: "Logs",
-              color: "grey",
-              subcategories: ["Unused"],
-            },
-            {
-              name: "Ruby",
-              color: "red",
-              subcategories: ["Code"],
-            },
-            {
-              name: "Native",
-              color: "blue",
-              subcategories: ["Code"],
-            },
-            {
-              name: "Native",
-              color: "lightblue",
-              subcategories: ["Code"],
-            },
-          ],
-          marker_schema: [],
-        },
-        libs: [],
-        counters: [],
-        threads: @profile[:threads].values.map {|th| ThreadReport.new(th).emit }
-      }
-      Reporter.deep_camelize_keys(report)
-    end
-
-    class ThreadReport
-      def initialize(thread)
-        @thread = thread
-
-        # Populated in other methods
-        @func_id_map = {}
-        @frame_id_map = {}
-        @stack_tree_id_map = {}
-
-        @string_table = {}
-      end
-
-      def inspect
-        "" # TODO: provide something better
-      end
-
-      def emit
-        x = weave_native_stack(@thread[:stack_tree])
-        @thread[:stack_tree] = x
-        func_table = build_func_table
-        frame_table = build_frame_table
-        stack_table = build_stack_table(func_table, frame_table)
-        samples = build_samples
-
-        string_table = build_string_table
-
-        {
-          process_type: 'default',
-          process_name: 'ruby',
-          process_startup_time: 0,
-          process_shutdown_time: nil,
-          register_time: 0,
-          unregister_time: nil,
-          paused_ranges: [],
-          name: "Thread (tid: #{@thread[:thread_id]})",
-          is_main_thread: true,
-          is_js_tracer: true,
-          # FIXME: We can fill the correct PID only after we correctly fill is_main_thread
-          # (only one thread could be marked as is_main_thread in a single process)
-          pid: @thread[:thread_id],
-          tid: @thread[:thread_id],
-          samples: samples,
-          markers: markers,
-          stack_table: stack_table,
-          frame_table: frame_table,
-          string_array: build_string_table,
-          func_table: func_table,
-          resource_table: {
-            lib: [],
-            name: [],
-            host: [],
-            type: [],
-            length: 0,
-          },
-          native_symbols: [],
-        }
-      end
-
-      def build_samples
-        ret = {
-          event_delay: [],
-          stack: [],
-          time: [],
-          duration: [],
-          # weight: nil,
-          # weight_type: 'samples',
-        }
-
-        @thread[:samples].each do |sample|
-          ret[:stack] << @stack_tree_id_map[sample[:stack_tree_id]]
-          ret[:time] << sample[:elapsed_ns] / 1000000 # ns -> ms
-          ret[:duration] << 1
-          ret[:event_delay] << 0
-        end
-
-        ret[:length] = ret[:stack].length
-        ret
-      end
-
-      def build_frame_table
-        ret = {
-          address: [],
-          category: [],
-          subcategory: [],
-          func: [],
-          inner_window_id: [],
-          implementation: [],
-          line: [],
-          column: [],
-          optimizations: [],
-          inline_depth: [],
-          native_symbol: [],
-        }
-
-        @thread[:frames].each.with_index do |(id, frame), i|
-          ret[:address] << frame[:address].to_s
-          ret[:category] << 1
-          ret[:subcategory] << 1
-          ret[:func] << i # TODO
-          ret[:inner_window_id] << nil
-          ret[:implementation] << nil
-          ret[:line] << frame[:callsite_lineno]
-          ret[:column] << nil
-          ret[:optimizations] << nil
-          ret[:inline_depth] << 0
-          ret[:native_symbol] << nil
-
-          @frame_id_map[id] = i
-        end
-
-        ret[:length] = ret[:address].length
-        ret
-      end
-
-      def build_func_table
-        ret = {
-          name: [],
-          is_js: [],
-          relevant_for_js: [],
-          resource: [],
-          file_name: [],
-          line_number: [],
-          column_number: [],
-        }
-
-        @thread[:frames].each.with_index do |(id, frame), i|
-          native = (frame[:entry_type] == 'Native')
-          label = "#{native ? "Native: " : ""}#{frame[:full_label]}"
-          ret[:name] << string_id(label)
-          ret[:is_js] << !native
-          ret[:relevant_for_js] << false
-          ret[:resource] << -1
-          ret[:file_name] << string_id(frame[:file_name])
-          ret[:line_number] << frame[:function_first_lineno]
-          ret[:column_number] << nil
-
-          @func_id_map[id] = i
-        end
-
-        ret[:length] = ret[:name].length
-        ret
-      end
-
-      # "Weave" the native stack into the Ruby stack.
-      #
-      # Strategy:
-      # - Split the stack into Ruby and Native parts
-      # - Start from the root of the Native stack
-      # - Dig in to the native stack until we hit a rb_vm_exec(), which marks a call into Ruby code
-      # - Switch to Ruby stack. Keep digging until we hit a Cfunc call, then switch back to Native stack
-      # - Repeat until we consume the entire stack
-      def weave_native_stack(stack_tree)
-        collected_paths = []
-        tree_to_array_of_paths(stack_tree, @thread[:frames], [], collected_paths)
-        collected_paths = collected_paths.map do |path|
-          next if path.size == 0
-
-          new_path = []
-          new_path << path.shift # root
-
-          # Split the stack into Ruby and Native parts
-          native_path, ruby_path = path.partition do |frame|
-            frame_id = frame[:frame_id]
-            @thread[:frames][frame_id][:entry_type] == 'Native'
-          end
-
-          mode = :native
-
-          loop do
-            break if ruby_path.size == 0 && native_path.size == 0
-
-            case mode
-            when :ruby
-              if ruby_path.size == 0
-                mode = :native
-                next
-              end
-
-              next_node = ruby_path[0]
-              new_path << ruby_path.shift
-              next_node_frame = @thread[:frames][next_node[:frame_id]]
-              if native_path.size > 0
-                # Search the remainder of the native stack for the same address
-                # Note: This isn't a very efficient way for the job... but it still works
-                ruby_addr = next_node_frame[:address]
-                native_path[0..].each do |native_node|
-                  native_addr = @thread[:frames][native_node[:frame_id]][:address]
-                  if ruby_addr && native_addr && ruby_addr == native_addr
-                    # A match has been found. Switch to native mode
-                    mode = :native
-                    break
-                  end
-                end
-              end
-            when :native
-              if native_path.size == 0
-                mode = :ruby
-                next
-              end
-
-              # Dig until we meet a rb_vm_exec
-              next_node = native_path[0]
-              new_path << native_path.shift
-              if @thread[:frames][next_node[:frame_id]][:full_label] =~ /vm_exec_core/ # VM_EXEC in vm_exec.h
-                mode = :ruby
-              end
-            end
-          end
-
-          new_path
-        end
-
-        # reconstruct stack_tree
-        new_stack_tree = array_of_paths_to_tree(collected_paths)
-        new_stack_tree
-      end
-
-      def tree_to_array_of_paths(stack_tree, frames, path, collected_paths)
-        new_path = path + [{ frame_id: stack_tree[:frame_id], node_id: stack_tree[:node_id] }]
-        if stack_tree[:children].empty?
-          collected_paths << new_path
-        else
-          stack_tree[:children].each do |frame_id, child|
-            tree_to_array_of_paths(child, frames, new_path, collected_paths)
-          end
-        end
-      end
-
-      def array_of_paths_to_tree(paths)
-        new_stack_tree = { children: {}, node_id: 0, frame_id: 0 }
-        paths.each do |path|
-          current = new_stack_tree
-          path[1..].each do |frame|
-            frame_id = frame[:frame_id]
-            node_id = frame[:node_id]
-            current[:children][frame_id] ||= { children: {}, node_id: node_id, frame_id: frame_id }
-            current = current[:children][frame_id]
-          end
-        end
-        new_stack_tree
-      end
-
-      def build_stack_table(func_table, frame_table)
-        ret = {
-          frame: [],
-          category: [],
-          subcategory: [],
-          prefix: [],
-        }
-
-        queue = []
-
-        @thread[:stack_tree][:children].each {|_, c| queue << [nil, c] }
-
-        loop do
-          break if queue.size == 0
-
-          prefix, node = queue.shift
-          ret[:frame] << @frame_id_map[node[:frame_id]]
-          ret[:category] << (build_string_table[func_table[:name][frame_table[:func][@frame_id_map[node[:frame_id]]]]].start_with?('Native:') ? 2 : 1)
-          ret[:subcategory] << nil
-          ret[:prefix] << prefix
-
-          # The index of this frame - children can refer to this frame using this index as prefix
-          frame_index = ret[:frame].length - 1
-          @stack_tree_id_map[node[:node_id]] = frame_index
-
-          # Enqueue children nodes
-          node[:children].each {|_, c| queue << [frame_index, c] }
-        end
-
-        ret[:length] = ret[:frame].length
-        ret
-      end
-
-      def build_string_table
-        @string_table.sort_by {|_, v| v}.map {|s| s[0] }
-      end
-
-      def string_id(str)
-        return @string_table[str] if @string_table.has_key?(str)
-        @string_table[str] = @string_table.length
-        @string_table[str]
-      end
-
-      def markers
-        {
-          data: [],
-          name: [],
-          time: [],
-          start_time: [],
-          end_time: [],
-          phase: [],
-          category: [],
-          length: 0
-        }
-      end
-    end
-
-    # Util functions
-    class << self
-      def snake_to_camel(s)
-        return "isJS" if s == "is_js"
-        return "relevantForJS" if s == "relevant_for_js"
-        return "innerWindowID" if s == "inner_window_id"
-        s.split('_').inject([]) {|buffer, p| buffer.push(buffer.size == 0 ? p : p.capitalize) }.join
-      end
-
-      def deep_transform_keys(value, &block)
-        case value
-        when Array
-          value.map {|v| deep_transform_keys(v, &block) }
-        when Hash
-          Hash[value.map {|k, v| [yield(k), deep_transform_keys(v, &block)] }]
-        else
-          value
-        end
-      end
-
-      def deep_camelize_keys(value)
-        deep_transform_keys(value) do |key|
-          snake_to_camel(key.to_s).to_sym
-        end
-      end
-
-      def deep_intize_keys(value)
-        deep_transform_keys(value) do |key|
-          if key.to_s.to_i.to_s == key.to_s
-            key.to_s.to_i
-          else
-            key
-          end
-        end
-      end
-    end
-  end
+  module Reporter; end
 end

data/lib/pf2/serve.rb

@@ -28,7 +28,7 @@ module Pf2
         profile = JSON.parse(profile, symbolize_names: true, max_nesting: false)
         res.header['Content-Type'] = 'application/json'
         res.header['Access-Control-Allow-Origin'] = '*'
-        res.body = JSON.generate(Pf2::Reporter.new((profile)).emit)
+        res.body = JSON.generate(Pf2::Reporter::FirefoxProfiler.new((profile)).emit)
         Pf2.start
       end
 

data/lib/pf2/version.rb

@@ -1,3 +1,3 @@
 module Pf2
-  VERSION = '0.5.2'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: pf2
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Daisuke Aritomo
 bindir: exe
 cert_chain: []
-date: 2024-07-12 00:00:00.000000000 Z
+date: 2024-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -171,6 +171,9 @@ files:
 - ext/pf2/src/ruby_internal_apis.rs
 - ext/pf2/src/sample.rs
 - ext/pf2/src/scheduler.rs
+- ext/pf2/src/serialization.rs
+- ext/pf2/src/serialization/profile.rs
+- ext/pf2/src/serialization/serializer.rs
 - ext/pf2/src/session.rs
 - ext/pf2/src/session/configuration.rs
 - ext/pf2/src/session/new_thread_watcher.rs
@@ -183,6 +186,7 @@ files:
 - lib/pf2.rb
 - lib/pf2/cli.rb
 - lib/pf2/reporter.rb
+- lib/pf2/reporter/firefox_profiler.rb
 - lib/pf2/serve.rb
 - lib/pf2/session.rb
 - lib/pf2/version.rb