vernier 0.8.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b450ebff29010a5a95e3817bc67d925f99ecca003ceacf15bdb390805e314045
-  data.tar.gz: c5c2ab7f3c3e29176685db787f9d4be9acbd5f798532ea571bd8710c78e05e62
+  metadata.gz: dd7670ec2d67c9d5e25c9a4359f3d02092737a6bb8a7e39910f880bf1ce16424
+  data.tar.gz: 3eb2bd5923128735f4a1a4d80de5b3400bb18134785333099f7b26091f3b9c91
 SHA512:
-  metadata.gz: fe6d2aaba11eb9955fa95ec2bb2d0aaf14945b0fc70184251fc46a763a45f0fa14ede8bf7e959aa43160b96531fa11efc9919a7352f2d703382d32310dd739a5
-  data.tar.gz: 6f0f28086118df8cefbb533a8ca822f14410f460e7fb933eed1655ecdbc07d19e823986c1035d9dfc10a728b6966fa9d90c474a60e4a2b4ac0dc9e02bda70719
+  metadata.gz: 01bef1081e96f0a03d325b5935c7fa083f053e9d40816b1eae26b6a23a46760169f1e9a1daa353e6c7a3cca0887a09b042fe8facc8b2158bc6282a1a0abc1305
+  data.tar.gz: ab12eedebac4ab40c7e853b824a0fc273c950f6e3ff136f40dba2910fa7acc77e7778f91e1b4197dbe0c4f6210f662f86fd13d6e898b930f6e683d164f355691

data/README.md

@@ -25,7 +25,7 @@ Rails benchmark - lobste.rs (time)
 Vernier requires Ruby version 3.2.1 or greater
 
 ```ruby
-gem 'vernier'
+gem "vernier", "~> 1.0"
 ```
 
 ## Usage

data/ext/vernier/vernier.cc

@@ -59,6 +59,8 @@ static VALUE rb_mVernierMarkerType;
 static VALUE rb_cVernierCollector;
 static VALUE rb_cStackTable;
 
+static VALUE sym_state, sym_gc_by;
+
 static const char *gvl_event_name(rb_event_flag_t event) {
     switch (event) {
       case RUBY_INTERNAL_THREAD_EVENT_STARTED:
@@ -796,6 +798,12 @@ static native_thread_id_t get_native_thread_id() {
 #endif
 }
 
+union MarkerInfo {
+    struct {
+        VALUE gc_by;
+        VALUE gc_state;
+    } gc_data;
+};
 
 class Marker {
     public:
@@ -833,8 +841,10 @@ class Marker {
     //native_thread_id_t thread_id;
     int stack_index = -1;
 
+    MarkerInfo extra_info;
+
     VALUE to_array() {
-        VALUE record[6] = {0};
+        VALUE record[7] = {0};
         record[0] = Qnil; // FIXME
         record[1] = INT2NUM(type);
         record[2] = INT2NUM(phase);
@@ -848,7 +858,14 @@ class Marker {
         }
         record[5] = stack_index == -1 ? Qnil : INT2NUM(stack_index);
 
-        return rb_ary_new_from_values(6, record);
+        if (type == Marker::MARKER_GC_PAUSE) {
+            VALUE hash = rb_hash_new();
+            record[6] = hash;
+
+            rb_hash_aset(hash, sym_gc_by, extra_info.gc_data.gc_by);
+            rb_hash_aset(hash, sym_state, extra_info.gc_data.gc_state);
+        }
+        return rb_ary_new_from_values(7, record);
     }
 };
 
@@ -863,7 +880,7 @@ class MarkerTable {
             list.push_back({ type, Marker::INTERVAL, from, to, stack_index });
         }
 
-        void record(Marker::Type type, int stack_index = -1) {
+        void record(Marker::Type type, int stack_index = -1, MarkerInfo extra_info = {}) {
             const std::lock_guard<std::mutex> lock(mutex);
 
             list.push_back({ type, Marker::INSTANT, TimeStamp::Now(), TimeStamp(), stack_index });
@@ -874,18 +891,41 @@ class GCMarkerTable: public MarkerTable {
     TimeStamp last_gc_entry;
 
     public:
+        void record_gc_start() {
+            record(Marker::Type::MARKER_GC_START);
+        }
+
         void record_gc_entered() {
           last_gc_entry = TimeStamp::Now();
         }
 
         void record_gc_leave() {
-          list.push_back({ Marker::MARKER_GC_PAUSE, Marker::INTERVAL, last_gc_entry, TimeStamp::Now(), -1 });
+            VALUE gc_state = rb_gc_latest_gc_info(sym_state);
+            VALUE gc_by = rb_gc_latest_gc_info(sym_gc_by);
+            union MarkerInfo info = {
+                .gc_data = {
+                    .gc_by = gc_by,
+                    .gc_state = gc_state
+                }
+            };
+            list.push_back({ Marker::MARKER_GC_PAUSE, Marker::INTERVAL, last_gc_entry, TimeStamp::Now(), -1, info });
+        }
+
+        void record_gc_end_mark() {
+            record_gc_leave();
+            record(Marker::Type::MARKER_GC_END_MARK);
+            record_gc_entered();
+        }
+
+        void record_gc_end_sweep() {
+            record(Marker::Type::MARKER_GC_END_SWEEP);
         }
 };
 
 enum Category{
-	CATEGORY_NORMAL,
-	CATEGORY_IDLE
+    CATEGORY_NORMAL,
+    CATEGORY_IDLE,
+    CATEGORY_STALLED
 };
 
 class ObjectSampleList {
@@ -1002,7 +1042,8 @@ class Thread {
             RUNNING,
             READY,
             SUSPENDED,
-            STOPPED
+            STOPPED,
+            INITIAL,
         };
 
         VALUE ruby_thread;
@@ -1058,12 +1099,14 @@ class Thread {
             }
 
             switch (new_state) {
+                case State::INITIAL:
+                    break;
                 case State::STARTED:
                     markers->record(Marker::Type::MARKER_GVL_THREAD_STARTED);
                     return; // no mutation of current state
                     break;
                 case State::RUNNING:
-                    assert(state == State::READY || state == State::RUNNING);
+                    assert(state == INITIAL || state == State::READY || state == State::RUNNING);
                     pthread_id = pthread_self();
                     native_tid = get_native_thread_id();
 
@@ -1081,7 +1124,7 @@ class Thread {
                     // Threads can be preempted, which means they will have been in "Running"
                     // state, and then the VM was like "no I need to stop you from working,
                     // so I'll put you in the 'ready' (or stalled) state"
-                    assert(state == State::STARTED || state == State::SUSPENDED || state == State::RUNNING);
+                    assert(state == State::INITIAL || state == State::STARTED || state == State::SUSPENDED || state == State::RUNNING);
                     if (state == State::SUSPENDED) {
                         markers->record_interval(Marker::Type::MARKER_THREAD_SUSPENDED, from, now, stack_on_suspend_idx);
                     }
@@ -1091,12 +1134,12 @@ class Thread {
                     break;
                 case State::SUSPENDED:
                     // We can go from RUNNING or STARTED to SUSPENDED
-                    assert(state == State::RUNNING || state == State::STARTED || state == State::SUSPENDED);
+                    assert(state == State::INITIAL || state == State::RUNNING || state == State::STARTED || state == State::SUSPENDED);
                     markers->record_interval(Marker::Type::MARKER_THREAD_RUNNING, from, now);
                     break;
                 case State::STOPPED:
                     // We can go from RUNNING or STARTED or SUSPENDED to STOPPED
-                    assert(state == State::RUNNING || state == State::STARTED || state == State::SUSPENDED);
+                    assert(state == State::INITIAL || state == State::RUNNING || state == State::STARTED || state == State::SUSPENDED);
                     markers->record_interval(Marker::Type::MARKER_THREAD_RUNNING, from, now);
                     markers->record(Marker::Type::MARKER_GVL_THREAD_EXITED);
 
@@ -1137,6 +1180,10 @@ class ThreadTable {
             }
         }
 
+        void initial(VALUE th) {
+            set_state(Thread::State::INITIAL, th);
+        }
+
         void started(VALUE th) {
             //list.push_back(Thread{pthread_self(), Thread::State::SUSPENDED});
             set_state(Thread::State::STARTED, th);
@@ -1172,7 +1219,7 @@ class ThreadTable {
             for (auto &threadptr : list) {
                 auto &thread = *threadptr;
                 if (thread_equal(th, thread.ruby_thread)) {
-                    if (new_state == Thread::State::SUSPENDED) {
+                    if (new_state == Thread::State::SUSPENDED || new_state == Thread::State::READY && (thread.state != Thread::State::SUSPENDED)) {
 
                         RawSample sample;
                         sample.sample();
@@ -1636,6 +1683,11 @@ class TimeCollector : public BaseCollector {
                             thread.stack_on_suspend_idx,
                             sample_start,
                             CATEGORY_IDLE);
+                } else if (thread.state == Thread::State::READY) {
+                    thread.samples.record_sample(
+                            thread.stack_on_suspend_idx,
+                            sample_start,
+                            CATEGORY_STALLED);
                 } else {
                 }
             }
@@ -1688,13 +1740,13 @@ class TimeCollector : public BaseCollector {
 
         switch (event) {
             case RUBY_INTERNAL_EVENT_GC_START:
-                collector->gc_markers.record(Marker::Type::MARKER_GC_START);
+                collector->gc_markers.record_gc_start();
                 break;
             case RUBY_INTERNAL_EVENT_GC_END_MARK:
-                collector->gc_markers.record(Marker::Type::MARKER_GC_END_MARK);
+                collector->gc_markers.record_gc_end_mark();
                 break;
             case RUBY_INTERNAL_EVENT_GC_END_SWEEP:
-                collector->gc_markers.record(Marker::Type::MARKER_GC_END_SWEEP);
+                collector->gc_markers.record_gc_end_sweep();
                 break;
             case RUBY_INTERNAL_EVENT_GC_ENTER:
                 collector->gc_markers.record_gc_entered();
@@ -1751,6 +1803,13 @@ class TimeCollector : public BaseCollector {
             return false;
         }
 
+        // Record one sample from each thread
+        VALUE all_threads = rb_funcall(rb_path2class("Thread"), rb_intern("list"), 0);
+        for (int i = 0; i < RARRAY_LEN(all_threads); i++) {
+            VALUE thread = RARRAY_AREF(all_threads, i);
+            this->threads.initial(thread);
+        }
+
         if (allocation_sample_rate > 0) {
             tp_newobj = rb_tracepoint_new(0, RUBY_INTERNAL_EVENT_NEWOBJ, newobj_i, this);
             rb_tracepoint_enable(tp_newobj);
@@ -1978,6 +2037,10 @@ Init_consts(VALUE rb_mVernierMarkerPhase) {
 extern "C" void
 Init_vernier(void)
 {
+    sym_state = sym("state");
+    sym_gc_by = sym("gc_by");
+    rb_gc_latest_gc_info(sym_state); // HACK: needs to be warmed so that it can be called during GC
+
   rb_mVernier = rb_define_module("Vernier");
   rb_cVernierResult = rb_define_class_under(rb_mVernier, "Result", rb_cObject);
   VALUE rb_mVernierMarker = rb_define_module_under(rb_mVernier, "Marker");

data/lib/vernier/autorun.rb

@@ -30,8 +30,8 @@ module Vernier
       result = @collector.stop
       @collector = nil
       output_path = options[:output]
-      output_path ||= Tempfile.create(["profile", ".vernier.json"]).path
-      File.write(output_path, Vernier::Output::Firefox.new(result).output)
+      output_path ||= Tempfile.create(["profile", ".vernier.json.gz"]).path
+      result.write(out: output_path)
 
       STDERR.puts(result.inspect)
       STDERR.puts("written to #{output_path}")
@@ -58,7 +58,7 @@ end
 if signal = Vernier::Autorun.options[:signal]
   STDERR.puts "to toggle profiler: kill -#{signal} #{Process.pid}"
   trap(signal) do
-    Vernier::Autorun.toggle
+    Thread.new { Vernier::Autorun.toggle }
   end
 end
 

data/lib/vernier/collector.rb

@@ -94,11 +94,12 @@ module Vernier
 
       marker_strings = Marker.name_table
 
-      markers = self.markers.map do |(tid, type, phase, ts, te, stack)|
+      markers = self.markers.map do |(tid, type, phase, ts, te, stack, extra_info)|
         name = marker_strings[type]
         sym = Marker::MARKER_SYMBOLS[type]
         data = { type: sym }
         data[:cause] = { stack: stack } if stack
+        data.merge!(extra_info) if extra_info
         [tid, name, ts, te, phase, data]
       end
 

data/lib/vernier/middleware.rb

@@ -18,10 +18,10 @@ module Vernier
       result = Vernier.trace(interval:, allocation_sample_rate:, hooks: [:rails]) do
         @app.call(env)
       end
-      body = result.to_gecko
-      filename = "#{request.path.gsub("/", "_")}_#{DateTime.now.strftime("%Y-%m-%d-%H-%M-%S")}.vernier.json"
+      body = result.to_gecko(gzip: true)
+      filename = "#{request.path.gsub("/", "_")}_#{DateTime.now.strftime("%Y-%m-%d-%H-%M-%S")}.vernier.json.gz"
       headers = {
-        "Content-Type" => "application/json; charset=utf-8",
+        "Content-Type" => "application/octet-stream",
         "Content-Disposition" => "attachment; filename=\"#{filename}\"",
         "Content-Length" => body.bytesize.to_s
       }

data/lib/vernier/output/firefox.rb

@@ -14,35 +14,29 @@ module Vernier
           @categories = []
           @categories_by_name = {}
 
-          add_category(name: "Default", color: "grey")
+          add_category(name: "Ruby", color: "grey") do |c|
+            rails_components = %w[ activesupport activemodel activerecord
+              actionview actionpack activejob actionmailer actioncable
+              activestorage actionmailbox actiontext railties ]
+            c.add_subcategory(
+              name: "Rails",
+              matcher: gem_path(*rails_components)
+            )
+            c.add_subcategory(
+              name: "gem",
+              matcher: starts_with(*Gem.path)
+            )
+            c.add_subcategory(
+              name: "stdlib",
+              matcher: starts_with(RbConfig::CONFIG["rubylibdir"])
+            )
+          end
           add_category(name: "Idle", color: "transparent")
+          add_category(name: "Stalled", color: "transparent")
 
           add_category(name: "GC", color: "red")
-          add_category(
-            name: "stdlib",
-            color: "red",
-            matcher: starts_with(RbConfig::CONFIG["rubylibdir"])
-          )
           add_category(name: "cfunc", color: "yellow", matcher: "<cfunc>")
 
-          rails_components = %w[ activesupport activemodel activerecord
-          actionview actionpack activejob actionmailer actioncable
-          activestorage actionmailbox actiontext railties ]
-          add_category(
-            name: "Rails",
-            color: "green",
-            matcher: gem_path(*rails_components)
-          )
-          add_category(
-            name: "gem",
-            color: "red",
-            matcher: starts_with(*Gem.path)
-          )
-          add_category(
-            name: "Application",
-            color: "purple"
-          )
-
           add_category(name: "Thread", color: "grey")
         end
 
@@ -50,6 +44,8 @@ module Vernier
           category = Category.new(@categories.length, name: name, **kw)
           @categories << category
           @categories_by_name[name] = category
+          category.add_subcategory(name: "Other")
+          yield category if block_given?
           category
         end
 
@@ -70,12 +66,17 @@ module Vernier
         end
 
         class Category
-          attr_reader :idx, :name, :color, :matcher
+          attr_reader :idx, :name, :color, :matcher, :subcategories
           def initialize(idx, name:, color:, matcher: nil)
             @idx = idx
             @name = name
             @color = color
             @matcher = matcher
+            @subcategories = []
+          end
+
+          def add_subcategory(**args)
+            @subcategories << Category.new(@subcategories.size, color: nil, **args)
           end
 
           def matches?(path)
@@ -89,8 +90,13 @@ module Vernier
         @categorizer = Categorizer.new
       end
 
-      def output
-        ::JSON.fast_generate(data)
+      def output(gzip: false)
+        result = ::JSON.fast_generate(data)
+        if gzip
+          require "zlib"
+          result = Zlib.gzip(result)
+        end
+        result
       end
 
       private
@@ -132,7 +138,7 @@ module Vernier
               {
                 name: category.name,
                 color: category.color,
-                subcategories: []
+                subcategories: category.subcategories.map(&:name)
               }
             end,
             sourceCodeIsNotOnSearchfox: true
@@ -188,7 +194,9 @@ module Vernier
               {
                 label: "Description",
                 value: "All threads are paused as GC is performed"
-              }
+              },
+              { key: "state", format: "string" },
+              { key: "gc_by", format: "string" },
             ]
           },
           *hook_additions
@@ -251,12 +259,22 @@ module Vernier
             func_implementations[func_idx]
           end
 
+          cfunc_category = @categorizer.get_category("cfunc")
+          ruby_category = @categorizer.get_category("Ruby")
           func_categories = filenames.map do |filename|
-            @categorizer.categorize(filename)
+            filename == "<cfunc>" ? cfunc_category : ruby_category
+          end
+          func_subcategories = filenames.map do |filename|
+            next 0 if filename == "<cfunc>"
+
+            (ruby_category.subcategories.detect {|c| c.matches?(filename) } || ruby_category.subcategories.first).idx
           end
           @frame_categories = profile.frame_table.fetch(:func).map do |func_idx|
             func_categories[func_idx]
           end
+          @frame_subcategories = profile.frame_table.fetch(:func).map do |func_idx|
+            func_subcategories[func_idx]
+          end
         end
 
         def filter_filenames(filenames)
@@ -412,11 +430,13 @@ module Vernier
           frames = profile.stack_table.fetch(:frame).dup
           prefixes = profile.stack_table.fetch(:parent).dup
           categories  = frames.map{|idx| @frame_categories[idx].idx }
+          subcategories  = frames.map{|idx| @frame_subcategories[idx] }
 
-          @categorized_stacks.keys.each do |(stack, category)|
+          @categorized_stacks.each_key do |(stack, category)|
             frames << frames[stack]
             prefixes << prefixes[stack]
             categories << category
+            subcategories << 0
           end
 
           size = frames.length
@@ -425,7 +445,7 @@ module Vernier
           {
             frame: frames,
             category: categories,
-            subcategory: [0] * size,
+            subcategory: subcategories,
             prefix: prefixes,
             length: prefixes.length
           }

data/lib/vernier/result.rb

@@ -37,12 +37,13 @@ module Vernier
       (current_time_real_ns - current_time_mono_ns + started_at_mono_ns)
     end
 
-    def to_gecko
-      Output::Firefox.new(self).output
+    def to_gecko(gzip: false)
+      Output::Firefox.new(self).output(gzip:)
     end
 
     def write(out:)
-      File.write(out, to_gecko)
+      gzip = out.end_with?(".gz")
+      File.binwrite(out, to_gecko(gzip:))
     end
 
     def elapsed_seconds
@@ -50,7 +51,7 @@ module Vernier
     end
 
     def inspect
-      "#<#{self.class} #{elapsed_seconds} seconds, #{threads.count} threads, #{weights.sum} samples, #{samples.uniq.size} unique>"
+      "#<#{self.class} #{elapsed_seconds} seconds, #{threads.count} threads, #{samples.count} samples, #{samples.uniq.size} unique>"
     end
 
     def each_sample

data/lib/vernier/stack_table.rb

@@ -1,5 +1,9 @@
 module Vernier
   class StackTable
+    def inspect
+      "#<#{self.class.name} #{stack_count} stacks, #{frame_count} frames, #{func_count} funcs>"
+    end
+
     def to_h
       {
         stack_table: {

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "0.8.0"
+  VERSION = "1.0.1"
 end

data/lib/vernier.rb

@@ -56,8 +56,8 @@ module Vernier
     result
   end
 
-  def self.trace_retained(out: nil, gc: true, &block)
-    profile(mode: :retained, out:, gc:, &block)
+  def self.trace_retained(**profile_options, &block)
+    profile(**profile_options.merge(mode: :retained), &block)
   end
 
   class Collector

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vernier
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.1
 platform: ruby
 authors:
 - John Hawthorn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-24 00:00:00.000000000 Z
+date: 2024-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport