vernier 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd7670ec2d67c9d5e25c9a4359f3d02092737a6bb8a7e39910f880bf1ce16424
-  data.tar.gz: 3eb2bd5923128735f4a1a4d80de5b3400bb18134785333099f7b26091f3b9c91
+  metadata.gz: aa7e8ccc4ac1312ccffb820ec0c6690209725a26ff0f5a9945a508edca073975
+  data.tar.gz: a0f7b10b284dd8fd387a8b12e15a2c9e8535e07f9d19f37c0734b08c93693956
 SHA512:
-  metadata.gz: 01bef1081e96f0a03d325b5935c7fa083f053e9d40816b1eae26b6a23a46760169f1e9a1daa353e6c7a3cca0887a09b042fe8facc8b2158bc6282a1a0abc1305
-  data.tar.gz: ab12eedebac4ab40c7e853b824a0fc273c950f6e3ff136f40dba2910fa7acc77e7778f91e1b4197dbe0c4f6210f662f86fd13d6e898b930f6e683d164f355691
+  metadata.gz: 7effed8c46b5110437f12d3fb8a4c4707e334c7edd51d4838198bb8dfb56cdd48c6d1700464be2d32b979f97eafc4c45a2406a4028cceadcac3bd71c1520babb
+  data.tar.gz: 4f11e253e293fd455d1eff820262fb6ea217bf0e3c574dd14bc15c3638e576fd3fb25ee84c3cdeb7151b2877d3f918b4157262f3268797482c192f3935b8768a

data/README.md

@@ -8,10 +8,10 @@ Next-generation Ruby 3.2.1+ sampling profiler. Tracks multiple threads, GVL acti
 
 [Livestreamed demo: Pairin' with Aaron (YouTube)](https://www.youtube.com/watch?v=9nvX3OHykGQ#t=27m43)
 
-Sidekiq jobs from Mastodon (time, threded)
+Sidekiq jobs from Mastodon (time, threaded)
 : https://share.firefox.dev/44jZRf3
 
-Puma web requests from Mastodon (time, threded)
+Puma web requests from Mastodon (time, threaded)
 : https://share.firefox.dev/48FOTnF
 
 Rails benchmark - lobste.rs (time)
@@ -50,16 +50,34 @@ starting profiler with interval 500
 written to /tmp/profile20240328-82441-gkzffc.vernier.json
 ```
 
-### Block of code
+#### Block of code
 
 ``` ruby
-Vernier.run(out: "time_profile.json") do
+Vernier.profile(out: "time_profile.json") do
   some_slow_method
 end
 ```
 
+Alternatively you can use the aliases `Vernier.run` and `Vernier.trace`.
+
+#### Start and stop
+
+```ruby
+Vernier.start_profile(out: "time_profile.json")
+
+some_slow_method
+
+# some other file
+
+some_other_slow_method
+
+Vernier.stop_profile
+```
+
 ### Retained memory
 
+#### Block of code
+
 Record a flamegraph of all **retained** allocations from loading `irb`.
 
 ```
@@ -68,6 +86,16 @@ ruby -r vernier -e 'Vernier.trace_retained(out: "irb_profile.json") { require "i
 
 Retained-memory flamegraphs must be interpreted a little differently than a typical profiling flamegraph. In a retained-memory flamegraph, the x-axis represents a proportion of memory in bytes,  _not time or samples_ The topmost boxes on the y-axis represent the retained objects, with their stacktrace below; their width represents the percentage of overall retained memory each object occupies.
 
+### Options
+
+Option | Description
+:- | :-
+`mode` | The sampling mode to use. One of `:wall`, `:retained` or `:custom`. Default is `:wall`.
+`out` | The file to write the profile to.
+`interval` | The sampling interval in microseconds. Default is `500`. Only available in `:wall` mode.
+`allocation_sample_rate` | The allocation sampling interval in number of allocations. Default is `0` (disabled). Only available in `:wall` mode.
+`gc` | Initiate a full and immediate garbage collection cycle before profiling. Default is `true`. Only available in `:retained` mode.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/examples/gvl_sleep.rb

@@ -1,34 +1,19 @@
 # Different (bad) ways to sleep
 
-File.write("#{__dir__}/my_sleep.c", <<~EOF)
-#include <time.h>
-#include <sys/errno.h>
+require 'bundler/inline'
 
-void my_sleep() {
-  struct timespec ts;
-  ts.tv_sec = 1;
-  ts.tv_nsec = 0;
+gemfile do
+  source 'https://rubygems.org'
 
-  int rc;
-  do {
-    rc = nanosleep(&ts, &ts);
-  } while (rc < 0 && errno == EINTR);
-}
-EOF
-
-soext = RbConfig::CONFIG["SOEXT"]
-system("gcc", "-shared", "-fPIC", "#{__dir__}/my_sleep.c", "-o", "#{__dir__}/my_sleep.#{soext}")
-
-require "fiddle"
-
-SLEEP_LIB = Fiddle.dlopen("./my_sleep.#{soext}")
+  gem "gvltest"
+end
 
 def cfunc_sleep_gvl
-  Fiddle::Function.new(SLEEP_LIB['my_sleep'], [], Fiddle::TYPE_VOID, need_gvl: true).call
+  GVLTest.sleep_holding_gvl(1)
 end
 
 def cfunc_sleep_idle
-  Fiddle::Function.new(SLEEP_LIB['my_sleep'], [], Fiddle::TYPE_VOID, need_gvl: true).call
+  GVLTest.sleep_without_gvl(1)
 end
 
 def ruby_sleep_gvl

data/ext/vernier/vernier.cc

@@ -14,9 +14,13 @@
 
 #include <sys/time.h>
 #include <signal.h>
-#ifdef __APPLE__
+#if defined(__APPLE__)
 /* macOS */
 #include <dispatch/dispatch.h>
+#elif defined(__FreeBSD__)
+/* FreeBSD */
+#include <pthread_np.h>
+#include <semaphore.h>
 #else
 /* Linux */
 #include <semaphore.h>
@@ -563,6 +567,20 @@ struct StackTable {
         }
     }
 
+    StackNode *convert_stack(StackTable &other, int original_idx) {
+        if (original_idx < 0) {
+            return &root_stack_node;
+        }
+
+        StackNode &original_node = other.stack_node_list[original_idx];
+        StackNode *parent_node = convert_stack(other, original_node.parent);
+        StackNode *node = next_stack_node(parent_node, original_node.frame);
+
+        return node;
+    }
+
+    static VALUE stack_table_convert(VALUE self, VALUE other, VALUE original_stack);
+
     static VALUE stack_table_stack_count(VALUE self);
     static VALUE stack_table_frame_count(VALUE self);
     static VALUE stack_table_func_count(VALUE self);
@@ -661,6 +679,32 @@ StackTable::stack_table_stack_count(VALUE self) {
     return INT2NUM(count);
 }
 
+VALUE
+StackTable::stack_table_convert(VALUE self, VALUE original_tableval, VALUE original_idxval) {
+    StackTable *stack_table = get_stack_table(self);
+    StackTable *original_table = get_stack_table(original_tableval);
+    int original_idx = NUM2INT(original_idxval);
+
+    int original_size;
+    {
+        const std::lock_guard<std::mutex> lock(original_table->stack_mutex);
+        original_size = original_table->stack_node_list.size();
+    }
+
+    if (original_idx >= original_size || original_idx < 0) {
+        rb_raise(rb_eRangeError, "index out of range");
+    }
+
+    int result_idx;
+    {
+        const std::lock_guard<std::mutex> lock1(stack_table->stack_mutex);
+        const std::lock_guard<std::mutex> lock2(original_table->stack_mutex);
+        StackNode *node = stack_table->convert_stack(*original_table, original_idx);
+        result_idx = node->index;
+    }
+    return INT2NUM(result_idx);
+}
+
 VALUE
 StackTable::stack_table_frame_count(VALUE self) {
     StackTable *stack_table = get_stack_table(self);
@@ -786,11 +830,13 @@ class SampleTranslator {
 
 typedef uint64_t native_thread_id_t;
 static native_thread_id_t get_native_thread_id() {
-#ifdef __APPLE__
+#if defined(__APPLE__)
     uint64_t thread_id;
     int e = pthread_threadid_np(pthread_self(), &thread_id);
     if (e != 0) rb_syserr_fail(e, "pthread_threadid_np");
     return thread_id;
+#elif defined(__FreeBSD__)
+    return pthread_getthreadid_np();
 #else
     // gettid() is only available as of glibc 2.30
     pid_t tid = syscall(SYS_gettid);
@@ -1156,6 +1202,10 @@ class Thread {
             return rb_thread_main() == ruby_thread;
         }
 
+        bool is_start(VALUE start_thread) {
+            return start_thread == ruby_thread;
+        }
+
         bool running() {
             return state != State::STOPPED;
         }
@@ -1263,6 +1313,7 @@ class BaseCollector {
     StackTable *stack_table;
     VALUE stack_table_value;
 
+    VALUE start_thread;
     TimeStamp started_at;
 
     BaseCollector(VALUE stack_table_value) : stack_table_value(stack_table_value), stack_table(get_stack_table(stack_table_value)) {
@@ -1274,6 +1325,7 @@ class BaseCollector {
             return false;
         }
 
+        start_thread = rb_thread_current();
         started_at = TimeStamp::Now();
 
         running = true;
@@ -1289,17 +1341,19 @@ class BaseCollector {
         return Qnil;
     }
 
-    void write_meta(VALUE result) {
-        VALUE meta = rb_hash_new();
-        rb_ivar_set(result, rb_intern("@meta"), meta);
+    virtual void write_meta(VALUE meta, VALUE result) {
         rb_hash_aset(meta, sym("started_at"), ULL2NUM(started_at.nanoseconds()));
+        rb_hash_aset(meta, sym("interval"), Qnil);
+        rb_hash_aset(meta, sym("allocation_sample_rate"), Qnil);
 
     }
 
     virtual VALUE build_collector_result() {
         VALUE result = rb_obj_alloc(rb_cVernierResult);
 
-        write_meta(result);
+        VALUE meta = rb_hash_new();
+        rb_ivar_set(result, rb_intern("@meta"), meta);
+        write_meta(meta, result);
 
         return result;
     }
@@ -1616,6 +1670,13 @@ class TimeCollector : public BaseCollector {
 
     }
 
+    void write_meta(VALUE meta, VALUE result) {
+        BaseCollector::write_meta(meta, result);
+        rb_hash_aset(meta, sym("interval"), ULL2NUM(interval.microseconds()));
+        rb_hash_aset(meta, sym("allocation_sample_rate"), ULL2NUM(allocation_sample_rate));
+
+    }
+
     private:
 
     void record_sample(const RawSample &sample, TimeStamp time, Thread &thread, Category category) {
@@ -1883,6 +1944,7 @@ class TimeCollector : public BaseCollector {
                 rb_hash_aset(hash, sym("stopped_at"), ULL2NUM(thread->stopped_at.nanoseconds()));
             }
             rb_hash_aset(hash, sym("is_main"), thread->is_main() ? Qtrue : Qfalse);
+            rb_hash_aset(hash, sym("is_start"), thread->is_start(BaseCollector::start_thread) ? Qtrue : Qfalse);
 
         }
 
@@ -2060,6 +2122,7 @@ Init_vernier(void)
   rb_undef_alloc_func(rb_cStackTable);
   rb_define_singleton_method(rb_cStackTable, "new", stack_table_new, 0);
   rb_define_method(rb_cStackTable, "current_stack", stack_table_current_stack, -1);
+  rb_define_method(rb_cStackTable, "convert", StackTable::stack_table_convert, 2);
   rb_define_method(rb_cStackTable, "stack_parent_idx", stack_table_stack_parent_idx, 1);
   rb_define_method(rb_cStackTable, "stack_frame_idx", stack_table_stack_frame_idx, 1);
   rb_define_method(rb_cStackTable, "frame_line_no", StackTable::stack_table_frame_line_no, 1);

data/lib/vernier/collector.rb

@@ -6,9 +6,8 @@ require_relative "thread_names"
 module Vernier
   class Collector
     def initialize(mode, options = {})
-      if options.fetch(:gc, true) && (mode == :retained)
-        GC.start
-      end
+      @gc = options.fetch(:gc, true) && (mode == :retained)
+      GC.start if @gc
 
       @mode = mode
       @out = options[:out]
@@ -75,7 +74,11 @@ module Vernier
     def stop
       result = finish
 
-      result.instance_variable_set("@stack_table", stack_table.to_h)
+      result.meta[:mode] = @mode
+      result.meta[:out] = @out
+      result.meta[:gc] = @gc
+
+      result.stack_table = stack_table
       @thread_names.finish
 
       @hooks.each do |hook|

data/lib/vernier/middleware.rb

@@ -12,8 +12,8 @@ module Vernier
       permitted = @permit.call(request)
       return @app.call(env) unless permitted
 
-      interval = request.GET.fetch(:vernier_interval, 200).to_i
-      allocation_sample_rate = request.GET.fetch(:vernier_allocation_sample_rate, 200).to_i
+      interval = request.GET.fetch("vernier_interval", 200).to_i
+      allocation_sample_rate = request.GET.fetch("vernier_allocation_sample_rate", 200).to_i
 
       result = Vernier.trace(interval:, allocation_sample_rate:, hooks: [:rails]) do
         @app.call(env)

data/lib/vernier/output/firefox.rb

@@ -106,7 +106,7 @@ module Vernier
       def data
         markers_by_thread = profile.markers.group_by { |marker| marker[0] }
 
-        thread_data = profile.threads.map do |ruby_thread_id, thread_info|
+        threads = profile.threads.map do |ruby_thread_id, thread_info|
           markers = markers_by_thread[ruby_thread_id] || []
           Thread.new(
             ruby_thread_id,
@@ -114,7 +114,7 @@ module Vernier
             @categorizer,
             markers: markers,
             **thread_info,
-          ).data
+          )
         end
 
         {
@@ -141,10 +141,12 @@ module Vernier
                 subcategories: category.subcategories.map(&:name)
               }
             end,
-            sourceCodeIsNotOnSearchfox: true
+            sourceCodeIsNotOnSearchfox: true,
+            initialVisibleThreads: threads.each_index.to_a,
+            initialSelectedThreads: Array(threads.find_index(&:is_start))
           },
           libs: [],
-          threads: thread_data
+          threads: threads.map(&:data)
         }
       end
 
@@ -204,9 +206,9 @@ module Vernier
       end
 
       class Thread
-        attr_reader :profile
+        attr_reader :profile, :is_start
 
-        def initialize(ruby_thread_id, profile, categorizer, name:, tid:, samples:, weights:, timestamps: nil, sample_categories: nil, markers:, started_at:, stopped_at: nil, allocations: nil, is_main: nil)
+        def initialize(ruby_thread_id, profile, categorizer, name:, tid:, samples:, weights:, timestamps: nil, sample_categories: nil, markers:, started_at:, stopped_at: nil, allocations: nil, is_main: nil, is_start: nil)
           @ruby_thread_id = ruby_thread_id
           @profile = profile
           @categorizer = categorizer
@@ -218,18 +220,32 @@ module Vernier
             @is_main = @ruby_thread_id == ::Thread.main.object_id
           end
           @is_main = true if profile.threads.size == 1
+          @is_start = is_start.nil? ? @is_main : is_start
+
+          @stack_table = Vernier::StackTable.new
+          samples = samples.map { |sample| @stack_table.convert(profile._stack_table, sample) }
+
+          @samples = samples
 
           timestamps ||= [0] * samples.size
-          @samples, @weights, @timestamps = samples, weights, timestamps
+          @weights, @timestamps = weights, timestamps
           @sample_categories = sample_categories || ([0] * samples.size)
-          @markers = markers
+          @markers = markers.map do |marker|
+            if stack_idx = marker[5]&.dig(:cause, :stack)
+              marker = marker.dup
+              new_idx = @stack_table.convert(profile._stack_table, stack_idx)
+              marker[5] = marker[5].merge({ cause: { stack: new_idx }})
+            end
+            marker
+          end
 
           @started_at, @stopped_at = started_at, stopped_at
 
-          names = profile.func_table.fetch(:name)
-          filenames = profile.func_table.fetch(:filename)
+          @stack_table_hash = @stack_table.to_h
+          names = @stack_table_hash[:func_table].fetch(:name)
+          filenames = @stack_table_hash[:func_table].fetch(:filename)
 
-          stacks_size = profile.stack_table.fetch(:frame).size
+          stacks_size = @stack_table.stack_count
           @categorized_stacks = Hash.new do |h, k|
             h[k] = h.size + stacks_size
           end
@@ -243,8 +259,6 @@ module Vernier
             @strings[filename]
           end
 
-          lines = profile.frame_table.fetch(:line)
-
           func_implementations = filenames.map do |filename|
             # Must match strings in `src/profile-logic/profile-data.js`
             # inside the firefox profiler. See `getFriendlyStackTypeName`
@@ -255,24 +269,27 @@ module Vernier
               nil
             end
           end
-          @frame_implementations = profile.frame_table.fetch(:func).map do |func_idx|
+          @frame_implementations = @stack_table_hash[:frame_table].fetch(:func).map do |func_idx|
             func_implementations[func_idx]
           end
 
           cfunc_category = @categorizer.get_category("cfunc")
           ruby_category = @categorizer.get_category("Ruby")
-          func_categories = filenames.map do |filename|
-            filename == "<cfunc>" ? cfunc_category : ruby_category
-          end
-          func_subcategories = filenames.map do |filename|
-            next 0 if filename == "<cfunc>"
-
-            (ruby_category.subcategories.detect {|c| c.matches?(filename) } || ruby_category.subcategories.first).idx
+          func_categories, func_subcategories = [], []
+          filenames.each do |filename|
+            if filename == "<cfunc>"
+              func_categories << cfunc_category
+              func_subcategories << 0
+            else
+              func_categories << ruby_category
+              subcategory = ruby_category.subcategories.detect {|c| c.matches?(filename) }&.idx || 0
+              func_subcategories << subcategory
+            end
           end
-          @frame_categories = profile.frame_table.fetch(:func).map do |func_idx|
+          @frame_categories = @stack_table_hash[:frame_table].fetch(:func).map do |func_idx|
             func_categories[func_idx]
           end
-          @frame_subcategories = profile.frame_table.fetch(:func).map do |func_idx|
+          @frame_subcategories = @stack_table_hash[:frame_table].fetch(:func).map do |func_idx|
             func_subcategories[func_idx]
           end
         end
@@ -420,15 +437,14 @@ module Vernier
             stack: samples,
             time: times,
             weight: weights,
-            weightType: "samples",
-            #weightType: "bytes",
+            weightType: profile.meta[:mode] == :retained ? "bytes" : "samples",
             length: samples.length
           }
         end
 
         def stack_table
-          frames = profile.stack_table.fetch(:frame).dup
-          prefixes = profile.stack_table.fetch(:parent).dup
+          frames =   @stack_table_hash[:stack_table].fetch(:frame).dup
+          prefixes = @stack_table_hash[:stack_table].fetch(:parent).dup
           categories  = frames.map{|idx| @frame_categories[idx].idx }
           subcategories  = frames.map{|idx| @frame_subcategories[idx] }
 
@@ -452,8 +468,8 @@ module Vernier
         end
 
         def frame_table
-          funcs = profile.frame_table.fetch(:func)
-          lines = profile.frame_table.fetch(:line)
+          funcs = @stack_table_hash[:frame_table].fetch(:func)
+          lines = @stack_table_hash[:frame_table].fetch(:line)
           size = funcs.length
           none = [nil] * size
           categories = @frame_categories.map(&:idx)
@@ -480,7 +496,7 @@ module Vernier
 
           cfunc_idx = @strings["<cfunc>"]
           is_js = @filenames.map { |fn| fn != cfunc_idx }
-          line_numbers = profile.func_table.fetch(:first_line).map.with_index do |line, i|
+          line_numbers = @stack_table_hash[:func_table].fetch(:first_line).map.with_index do |line, i|
             if is_js[i] || line != 0
               line
             else

data/lib/vernier/result.rb

@@ -1,15 +1,11 @@
 module Vernier
   class Result
-    def stack_table
-      @stack_table[:stack_table]
+    def stack_table=(stack_table)
+      @stack_table = stack_table
     end
 
-    def frame_table
-      @stack_table[:frame_table]
-    end
-
-    def func_table
-      @stack_table[:func_table]
+    def _stack_table
+      @stack_table
     end
 
     attr_reader :markers
@@ -19,6 +15,7 @@ module Vernier
     attr_accessor :pid, :end_time
     attr_accessor :threads
     attr_accessor :meta
+    attr_accessor :mode
 
     def main_thread
       threads.values.detect {|x| x[:is_main] }
@@ -81,12 +78,12 @@ module Vernier
 
     class Func < BaseType
       def label
-        result.func_table[:name][idx]
+        result._stack_table.func_name(idx)
       end
       alias name label
 
       def filename
-        result.func_table[:filename][idx]
+        result._stack_table.func_filename(idx)
       end
 
       def to_s
@@ -100,12 +97,12 @@ module Vernier
       alias name label
 
       def func
-        func_idx = result.frame_table[:func][idx]
+        func_idx = result._stack_table.frame_func_idx(idx)
         Func.new(result, func_idx)
       end
 
       def line
-        result.frame_table[:line][idx]
+        result.frame_line_idx(idx)
       end
 
       def to_s
@@ -119,14 +116,14 @@ module Vernier
 
         stack_idx = idx
         while stack_idx
-          frame_idx = result.stack_table[:frame][stack_idx]
+          frame_idx = result._stack_table.stack_frame_idx(stack_idx)
           yield Frame.new(result, frame_idx)
-          stack_idx = result.stack_table[:parent][stack_idx]
+          stack_idx = result._stack_table.stack_parent_idx(stack_idx)
         end
       end
 
       def leaf_frame_idx
-        result.stack_table[:frame][idx]
+        result._stack_table.stack_frame_idx(idx)
       end
 
       def leaf_frame

data/lib/vernier/thread_names.rb

@@ -10,7 +10,7 @@ module Vernier
     end
 
     def [](object_id)
-      @names[object_id] || "unknown thread #{object_id}"
+      @names[object_id] || "thread obj_id:#{object_id}"
     end
 
     def finish

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

data/vernier.gemspec

@@ -31,5 +31,6 @@ Gem::Specification.new do |spec|
   spec.extensions = ["ext/vernier/extconf.rb"]
 
   spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "gvltest"
   spec.add_development_dependency "rack"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vernier
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - John Hawthorn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-08 00:00:00.000000000 Z
+date: 2024-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: gvltest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement