pf2 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 929c0f24675525d7d52bd0293ec45b9383579517119baac5b28da2bfe720d7d5
-  data.tar.gz: e75df94cf2085aea9d218b431e44ccda5f87e537a1c566ea67ae4ef04a03cb8a
+  metadata.gz: c4b713a9333a657f9b2259cdbae91e9aabde4312c2791a4dce517ce7d30cc46a
+  data.tar.gz: f01a941327ac6f2ce8b116a466a85a3021ffe0f612063b0f1466e7a31b373984
 SHA512:
-  metadata.gz: 2e28b5f4cbb99dc101b7f9a9092b1602d682f1001fa1c86a538d0d966a4c670df5fc5ecb971d8a25e093c3c3bdbe115b772ed40be50b8027cbc09fe6664c1db5
-  data.tar.gz: cf63fc413c60dda6fc0a360d39701cffc36aceeb98dc5d1d92a6e42761fa2ba1d2381e67a9a4af10fcb51fbf6e4b32afc89ff9307370fa542db278c6e76d40ae
+  metadata.gz: cf8c6b94d2a2ccee5912388a4f2f43f09612f9f3ba5a4ff10d615c7798bde30eb6061f03a9c79fa1ffe946da5f4a065eeae2bbe75ce1bb99821ce5253cc3ba9f
+  data.tar.gz: 50bd5c325fded53cf1585245469232c3fb184d1b3b893df0deb35a8f5d410046c3c821b1a27d468df6a3bb9ac8e2d7a12011aaa94a8d6889ca013dd4ec1f8eb1

data/CHANGELOG.md

@@ -1,6 +1,37 @@
 ## [Unreleased]
 
 
+## [0.5.0] - 2024-03-25
+
+### Added
+
+- `pf2 serve` subcommand
+  - `pf2 serve -- ruby target.rb`
+  - Profile programs without any change
+- New option: `threads: :all`
+  - When specified, Pf2 will track all active threads.
+  - `threads: nil` / omitting the `threads` option has the same effect.
+- Introduce `Pf2::Session` (https://github.com/osyoyu/pf2/pull/16)
+  - `Session` will be responsible for managing Profiles and Schedulers
+
+### Removed
+
+- `Pf2::SignalScheduler` and `Pf2::TimerThreadScheduler` are now hidden from Ruby.
+- `track_all_threads` option is removed in favor of `threads: :all`.
+
+
+## [0.4.0] - 2024-03-22
+
+### Added
+
+- New option: `track_all_threads`
+  - When true, all Threads will be tracked regardless of the `threads` option.
+
+### Removed
+
+- The `track_new_threads` option was removed in favor of the `track_all_threads` option.
+
+
 ## [0.3.0] - 2024-02-05
 
 ### Added

data/Cargo.lock

@@ -456,9 +456,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
 
 [[package]]
 name = "shlex"
-version = "1.2.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "syn"

data/README.md

@@ -8,10 +8,24 @@ Notable Capabilites
 
 - Can accurately track multiple Ruby Threads' activity
 - Sampling interval can be set based on per-Thread CPU usage
+- Can record native (C-level) stack traces side-by-side with Ruby traces
 
 Usage
 --------
 
+### Quickstart
+
+Run your Ruby program through `pf2 serve`.
+Wait a while until Pf2 collects profiles (or until the target program exits), then open the displayed link for visualization.
+
+```
+$ pf2 serve -- ruby target.rb
+[Pf2] Listening on localhost:51502.
+[Pf2] Open https://profiler.firefox.com/from-url/http%3A%2F%2Flocalhost%3A51502%2Fprofile for visualization.
+
+I'm the target program!
+```
+
 ### Profiling
 
 Pf2 will collect samples every 10 ms of wall time by default.
@@ -54,11 +68,10 @@ Pf2 accepts the following configuration keys:
 ```rb
 Pf2.start(
   interval_ms: 49,        # Integer: The sampling interval in milliseconds (default: 49)
-  threads: [],            # Array<Thread>: A list of Ruby Threads to be tracked (default: `Thread.list`)
   time_mode: :cpu,        # `:cpu` or `:wall`: The sampling timer's mode
                           # (default: `:cpu` for SignalScheduler, `:wall` for TimerThreadScheduler)
-  track_new_threads: true # Boolean: Whether to automatically track Threads spawned after profiler start
-                          # (default: false)
+  threads: [th1, th2],    # `Array<Thread>` | `:all`: A list of Ruby Threads to be tracked.
+                          # When `:all` or unspecified, Pf2 will track all active Threads.
 )
 ```
 

data/ext/pf2/src/backtrace.rs

@@ -42,6 +42,7 @@ impl Backtrace {
         }
     }
 
+    #[allow(dead_code)] // TODO: Remove this
     pub fn backtrace_pcinfo<F>(
         state: &BacktraceState,
         pc: usize,

data/ext/pf2/src/lib.rs

@@ -9,7 +9,11 @@ mod profile;
 mod profile_serializer;
 mod ringbuffer;
 mod sample;
+mod scheduler;
+mod session;
 #[cfg(target_os = "linux")]
 mod signal_scheduler;
 mod timer_thread_scheduler;
 mod util;
+
+mod ruby_internal_apis;

data/ext/pf2/src/profile_serializer.rs

@@ -6,6 +6,7 @@ use rb_sys::*;
 
 use crate::backtrace::Backtrace;
 use crate::profile::Profile;
+use crate::util::RTEST;
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct ProfileSerializer {
@@ -62,6 +63,10 @@ struct FrameTableEntry {
     id: FrameTableId,
     entry_type: FrameTableEntryType,
     full_label: String,
+    file_name: Option<String>,
+    function_first_lineno: Option<i32>,
+    callsite_lineno: Option<i32>,
+    address: Option<usize>,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -77,6 +82,11 @@ struct ProfileSample {
     stack_tree_id: StackTreeNodeId,
 }
 
+struct NativeFunctionFrame {
+    pub symbol_name: String,
+    pub address: Option<usize>,
+}
+
 impl ProfileSerializer {
     pub fn serialize(profile: &Profile) -> String {
         let mut sequence = 1;
@@ -92,49 +102,46 @@ impl ProfileSerializer {
 
                 // Process C-level stack
 
-                // A vec to keep the "programmer's" C stack trace.
-                // A single PC may be mapped to multiple inlined frames,
-                // so we keep the expanded stack frame in this Vec.
-                let mut c_stack: Vec<String> = vec![];
+                let mut c_stack: Vec<NativeFunctionFrame> = vec![];
+                // Rebuild the original backtrace (including inlined functions) from the PC.
                 for i in 0..sample.c_backtrace_pcs[0] {
                     let pc = sample.c_backtrace_pcs[i + 1];
                     Backtrace::backtrace_syminfo(
                         &profile.backtrace_state,
                         pc,
-                        |_pc: usize, symname: *const c_char, _symval: usize, _symsize: usize| {
+                        |_pc: usize, symname: *const c_char, symval: usize, _symsize: usize| {
                             if symname.is_null() {
-                                c_stack.push("(no symbol information)".to_owned());
+                                c_stack.push(NativeFunctionFrame {
+                                    symbol_name: "(no symbol information)".to_owned(),
+                                    address: None,
+                                });
                             } else {
-                                c_stack.push(CStr::from_ptr(symname).to_str().unwrap().to_owned());
+                                c_stack.push(NativeFunctionFrame {
+                                    symbol_name: CStr::from_ptr(symname)
+                                        .to_str()
+                                        .unwrap()
+                                        .to_owned(),
+                                    address: Some(symval),
+                                });
                             }
                         },
                         Some(Backtrace::backtrace_error_callback),
                     );
                 }
-
-                // Strip the C stack trace:
-                // - Remove Pf2-related frames which are always captured
-                // - Remove frames below rb_vm_exec
-                let mut reached_ruby = false;
-                c_stack.retain(|frame| {
-                    if reached_ruby {
-                        return false;
-                    }
-                    if frame.contains("pf2") {
-                        return false;
-                    }
-                    if frame.contains("rb_vm_exec") || frame.contains("vm_call_cfunc_with_frame") {
-                        reached_ruby = true;
-                        return false;
+                for frame in c_stack.iter() {
+                    if frame.symbol_name.contains("pf2") {
+                        // Skip Pf2-related frames
+                        continue;
                     }
-                    true
-                });
 
-                for frame in c_stack.iter() {
                     merged_stack.push(FrameTableEntry {
-                        id: calculate_id_for_c_frame(frame),
+                        id: calculate_id_for_c_frame(&frame.symbol_name),
                         entry_type: FrameTableEntryType::Native,
-                        full_label: frame.to_string(),
+                        full_label: frame.symbol_name.clone(),
+                        file_name: None,
+                        function_first_lineno: None,
+                        callsite_lineno: None,
+                        address: frame.address,
                     });
                 }
 
@@ -143,15 +150,52 @@ impl ProfileSerializer {
                 let ruby_stack_depth = sample.line_count;
                 for i in 0..ruby_stack_depth {
                     let frame: VALUE = sample.frames[i as usize];
+                    let lineno: i32 = sample.linenos[i as usize];
+                    let address: Option<usize> = {
+                        let cme = frame
+                            as *mut crate::ruby_internal_apis::rb_callable_method_entry_struct;
+                        let cme = &*cme;
+
+                        if (*(cme.def)).type_ == 1 {
+                            // The cme is a Cfunc
+                            Some((*(cme.def)).cfunc.func as usize)
+                        } else {
+                            // The cme is an ISeq (Ruby code) or some other type
+                            None
+                        }
+                    };
+                    let mut frame_full_label: VALUE = rb_profile_frame_full_label(frame);
+                    let frame_full_label: String = if RTEST(frame_full_label) {
+                        CStr::from_ptr(rb_string_value_cstr(&mut frame_full_label))
+                            .to_str()
+                            .unwrap()
+                            .to_owned()
+                    } else {
+                        "(unknown)".to_owned()
+                    };
+                    let mut frame_path: VALUE = rb_profile_frame_path(frame);
+                    let frame_path: String = if RTEST(frame_path) {
+                        CStr::from_ptr(rb_string_value_cstr(&mut frame_path))
+                            .to_str()
+                            .unwrap()
+                            .to_owned()
+                    } else {
+                        "(unknown)".to_owned()
+                    };
+                    let frame_first_lineno: VALUE = rb_profile_frame_first_lineno(frame);
+                    let frame_first_lineno: Option<i32> = if RTEST(frame_first_lineno) {
+                        Some(rb_num2int(frame_first_lineno).try_into().unwrap())
+                    } else {
+                        None
+                    };
                     merged_stack.push(FrameTableEntry {
                         id: frame,
                         entry_type: FrameTableEntryType::Ruby,
-                        full_label: CStr::from_ptr(rb_string_value_cstr(
-                            &mut rb_profile_frame_full_label(frame),
-                        ))
-                        .to_str()
-                        .unwrap()
-                        .to_owned(),
+                        full_label: frame_full_label,
+                        file_name: Some(frame_path),
+                        function_first_lineno: frame_first_lineno,
+                        callsite_lineno: Some(lineno),
+                        address,
                     });
                 }
 

data/ext/pf2/src/ruby_init.rs

@@ -2,9 +2,7 @@
 
 use rb_sys::*;
 
-#[cfg(target_os = "linux")]
-use crate::signal_scheduler::SignalScheduler;
-use crate::timer_thread_scheduler::TimerThreadScheduler;
+use crate::session::ruby_object::SessionRubyObject;
 use crate::util::*;
 
 #[allow(non_snake_case)]
@@ -21,53 +19,24 @@ extern "C" fn Init_pf2() {
     unsafe {
         let rb_mPf2: VALUE = rb_define_module(cstr!("Pf2"));
 
-        #[cfg(target_os = "linux")]
-        {
-            let rb_mPf2_SignalScheduler =
-                rb_define_class_under(rb_mPf2, cstr!("SignalScheduler"), rb_cObject);
-            rb_define_alloc_func(rb_mPf2_SignalScheduler, Some(SignalScheduler::rb_alloc));
-            rb_define_method(
-                rb_mPf2_SignalScheduler,
-                cstr!("initialize"),
-                Some(to_ruby_cfunc_with_args(SignalScheduler::rb_initialize)),
-                -1,
-            );
-            rb_define_method(
-                rb_mPf2_SignalScheduler,
-                cstr!("start"),
-                Some(to_ruby_cfunc_with_no_args(SignalScheduler::rb_start)),
-                0,
-            );
-            rb_define_method(
-                rb_mPf2_SignalScheduler,
-                cstr!("stop"),
-                Some(to_ruby_cfunc_with_no_args(SignalScheduler::rb_stop)),
-                0,
-            );
-        }
-
-        let rb_mPf2_TimerThreadScheduler =
-            rb_define_class_under(rb_mPf2, cstr!("TimerThreadScheduler"), rb_cObject);
-        rb_define_alloc_func(
-            rb_mPf2_TimerThreadScheduler,
-            Some(TimerThreadScheduler::rb_alloc),
-        );
+        let rb_mPf2_Session = rb_define_class_under(rb_mPf2, cstr!("Session"), rb_cObject);
+        rb_define_alloc_func(rb_mPf2_Session, Some(SessionRubyObject::rb_alloc));
         rb_define_method(
-            rb_mPf2_TimerThreadScheduler,
+            rb_mPf2_Session,
             cstr!("initialize"),
-            Some(to_ruby_cfunc_with_args(TimerThreadScheduler::rb_initialize)),
+            Some(to_ruby_cfunc_with_args(SessionRubyObject::rb_initialize)),
             -1,
         );
         rb_define_method(
-            rb_mPf2_TimerThreadScheduler,
+            rb_mPf2_Session,
             cstr!("start"),
-            Some(to_ruby_cfunc_with_no_args(TimerThreadScheduler::rb_start)),
+            Some(to_ruby_cfunc_with_no_args(SessionRubyObject::rb_start)),
             0,
         );
         rb_define_method(
-            rb_mPf2_TimerThreadScheduler,
+            rb_mPf2_Session,
             cstr!("stop"),
-            Some(to_ruby_cfunc_with_no_args(TimerThreadScheduler::rb_stop)),
+            Some(to_ruby_cfunc_with_no_args(SessionRubyObject::rb_stop)),
             0,
         );
     }

data/ext/pf2/src/ruby_internal_apis.rs

@@ -0,0 +1,70 @@
+#![allow(non_snake_case)]
+#![allow(non_camel_case_types)]
+
+use libc::{clockid_t, pthread_getcpuclockid, pthread_t};
+use rb_sys::{rb_check_typeddata, rb_data_type_struct, RTypedData, VALUE};
+use std::ffi::{c_char, c_int, c_void};
+use std::mem::MaybeUninit;
+
+// Types and structs from Ruby 3.4.0.
+
+#[repr(C)]
+pub struct rb_callable_method_entry_struct {
+    /* same fields with rb_method_entry_t */
+    pub flags: VALUE,
+    _padding_defined_class: VALUE,
+    pub def: *mut rb_method_definition_struct,
+    // ...
+}
+
+#[repr(C)]
+pub struct rb_method_definition_struct {
+    pub type_: c_int,
+    _padding: [c_char; 4],
+    pub cfunc: rb_method_cfunc_struct,
+    // ...
+}
+
+#[repr(C)]
+pub struct rb_method_cfunc_struct {
+    pub func: *mut c_void,
+    // ...
+}
+
+type rb_nativethread_id_t = libc::pthread_t;
+
+#[repr(C)]
+struct rb_native_thread {
+    _padding_serial: [c_char; 4], // rb_atomic_t
+    _padding_vm: *mut c_int,      // struct rb_vm_struct
+    thread_id: rb_nativethread_id_t,
+    // ...
+}
+
+#[repr(C)]
+struct rb_thread_struct {
+    _padding_lt_node: [c_char; 16], // struct ccan_list_node
+    _padding_self: VALUE,
+    _padding_ractor: *mut c_int, // rb_ractor_t
+    _padding_vm: *mut c_int,     // rb_vm_t
+    nt: *mut rb_native_thread,
+    // ...
+}
+type rb_thread_t = rb_thread_struct;
+
+/// Reimplementation of the internal RTYPEDDATA_TYPE macro.
+unsafe fn RTYPEDDATA_TYPE(obj: VALUE) -> *const rb_data_type_struct {
+    let typed: *mut RTypedData = obj as *mut RTypedData;
+    (*typed).type_
+}
+
+unsafe fn rb_thread_ptr(thread: VALUE) -> *mut rb_thread_t {
+    unsafe { rb_check_typeddata(thread, RTYPEDDATA_TYPE(thread)) as *mut rb_thread_t }
+}
+
+pub unsafe fn rb_thread_getcpuclockid(thread: VALUE) -> clockid_t {
+    let mut cid: clockid_t = MaybeUninit::zeroed().assume_init();
+    let pthread_id: pthread_t = (*(*rb_thread_ptr(thread)).nt).thread_id;
+    pthread_getcpuclockid(pthread_id, &mut cid as *mut clockid_t);
+    cid
+}

data/ext/pf2/src/scheduler.rs

@@ -0,0 +1,10 @@
+use rb_sys::{size_t, VALUE};
+
+pub trait Scheduler {
+    fn start(&self) -> VALUE;
+    fn stop(&self) -> VALUE;
+    fn on_new_thread(&self, thread: VALUE);
+    fn dmark(&self);
+    fn dfree(&self);
+    fn dsize(&self) -> size_t;
+}

data/ext/pf2/src/session/configuration.rs

@@ -0,0 +1,106 @@
+use std::collections::HashSet;
+use std::str::FromStr;
+use std::time::Duration;
+
+use rb_sys::*;
+
+use crate::util::cstr;
+
+pub const DEFAULT_SCHEDULER: Scheduler = Scheduler::Signal;
+pub const DEFAULT_INTERVAL: Duration = Duration::from_millis(49);
+pub const DEFAULT_TIME_MODE: TimeMode = TimeMode::CpuTime;
+
+#[derive(Clone, Debug)]
+pub struct Configuration {
+    pub scheduler: Scheduler,
+    pub interval: Duration,
+    pub time_mode: TimeMode,
+    pub target_ruby_threads: Threads,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Scheduler {
+    Signal,
+    TimerThread,
+}
+
+impl FromStr for Scheduler {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "signal" => Ok(Self::Signal),
+            "timer_thread" => Ok(Self::TimerThread),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum TimeMode {
+    CpuTime,
+    WallTime,
+}
+
+impl FromStr for TimeMode {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "cpu" => Ok(Self::CpuTime),
+            "wall" => Ok(Self::WallTime),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Threads {
+    All,
+    Targeted(HashSet<VALUE>),
+}
+
+impl Configuration {
+    pub fn validate(&self) -> Result<(), String> {
+        if self.scheduler == Scheduler::TimerThread && self.time_mode == TimeMode::CpuTime {
+            return Err("TimerThread scheduler does not support `time_mode: :cpu`.".to_owned());
+        }
+        if self.scheduler == Scheduler::TimerThread && self.target_ruby_threads == Threads::All {
+            return Err(concat!(
+                "TimerThread scheduler does not support `threads: :all` at the moment. ",
+                "Consider using `threads: Thread.list` for watching all threads at profiler start."
+            )
+            .to_owned());
+        }
+
+        Ok(())
+    }
+
+    pub fn to_rb_hash(&self) -> VALUE {
+        let hash: VALUE = unsafe { rb_hash_new() };
+        unsafe {
+            rb_hash_aset(
+                hash,
+                rb_id2sym(rb_intern(cstr!("scheduler"))),
+                rb_id2sym(rb_intern(match self.scheduler {
+                    Scheduler::Signal => cstr!("signal"),
+                    Scheduler::TimerThread => cstr!("timer_thread"),
+                })),
+            );
+            rb_hash_aset(
+                hash,
+                rb_id2sym(rb_intern(cstr!("interval_ms"))),
+                rb_int2inum(self.interval.as_millis().try_into().unwrap()),
+            );
+            rb_hash_aset(
+                hash,
+                rb_id2sym(rb_intern(cstr!("time_mode"))),
+                rb_id2sym(rb_intern(match self.time_mode {
+                    TimeMode::CpuTime => cstr!("cpu"),
+                    TimeMode::WallTime => cstr!("wall"),
+                })),
+            );
+        }
+        hash
+    }
+}

data/ext/pf2/src/session/new_thread_watcher.rs

@@ -0,0 +1,80 @@
+use std::collections::HashSet;
+use std::ffi::c_void;
+use std::mem::ManuallyDrop;
+use std::ptr::null_mut;
+use std::rc::Rc;
+use std::sync::Mutex;
+
+use rb_sys::*;
+
+/// A helper to watch new Ruby threads.
+///
+/// `NewThreadWatcher` operates on the Events Hooks API.
+/// Instead of relying on the `THREAD_EVENT_STARTED` event, it combines the
+/// `THREAD_EVENT_RESUMED` event and an internal _known-threads_ record.
+///
+/// This is to support operations requiring the underlying pthread. Ruby Threads
+/// are not guaranteed to be fully initialized at the time
+/// `THREAD_EVENT_STARTED` is triggered; i.e. the underlying pthread has not
+/// been created yet and `Thread#native_thread_id` returns `nil`.
+pub struct NewThreadWatcher {
+    inner: Rc<Mutex<Inner>>,
+    event_hook: *mut rb_internal_thread_event_hook_t,
+}
+
+struct Inner {
+    known_threads: HashSet<VALUE>,
+    on_new_thread: Box<dyn Fn(VALUE)>,
+}
+
+impl NewThreadWatcher {
+    pub fn watch<F>(callback: F) -> Self
+    where
+        F: Fn(VALUE) + 'static,
+    {
+        let mut watcher = Self {
+            inner: Rc::new(Mutex::new(Inner {
+                known_threads: HashSet::new(),
+                on_new_thread: Box::new(callback),
+            })),
+            event_hook: null_mut(),
+        };
+
+        let inner_ptr = Rc::into_raw(Rc::clone(&watcher.inner));
+        unsafe {
+            watcher.event_hook = rb_internal_thread_add_event_hook(
+                Some(Self::on_thread_resume),
+                RUBY_INTERNAL_THREAD_EVENT_RESUMED,
+                inner_ptr as *mut c_void,
+            );
+        };
+
+        watcher
+    }
+
+    unsafe extern "C" fn on_thread_resume(
+        _flag: rb_event_flag_t,
+        data: *const rb_internal_thread_event_data,
+        custom_data: *mut c_void,
+    ) {
+        let ruby_thread: VALUE = unsafe { (*data).thread };
+
+        // A pointer to Box<Inner> is passed as custom_data
+        let inner = unsafe { ManuallyDrop::new(Box::from_raw(custom_data as *mut Mutex<Inner>)) };
+        let mut inner = inner.lock().unwrap();
+
+        if !inner.known_threads.contains(&ruby_thread) {
+            inner.known_threads.insert(ruby_thread);
+            (inner.on_new_thread)(ruby_thread);
+        }
+    }
+}
+
+impl Drop for NewThreadWatcher {
+    fn drop(&mut self) {
+        log::trace!("Cleaning up event hook");
+        unsafe {
+            rb_internal_thread_remove_event_hook(self.event_hook);
+        }
+    }
+}