vernier 1.8.0 → 1.9.0

data/ext/vernier/vernier.hh

@@ -3,8 +3,15 @@
 
 #include "ruby.h"
 
+// HACK: This isn't public, but the objspace ext uses it
+extern "C" size_t rb_obj_memsize_of(VALUE);
+
+#define sym(name) ID2SYM(rb_intern_const(name))
+
 extern VALUE rb_mVernier;
 
 void Init_memory();
+void Init_stack_table();
+void Init_heap_tracker();
 
 #endif /* VERNIER_H */

data/lib/vernier/collector.rb

@@ -5,6 +5,110 @@ require_relative "thread_names"
 
 module Vernier
   class Collector
+    class CustomCollector < Collector
+      def initialize(mode, options)
+        @stack_table = StackTable.new
+
+        @samples = []
+        @timestamps = []
+
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        super
+      end
+
+      def sample
+        @samples << @stack_table.current_stack
+        @timestamps << Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+      end
+
+      def start
+      end
+
+      def finish
+        result = Result.new
+        result.instance_variable_set(:@threads, {
+          0 => {
+            tid: 0,
+            name: "custom",
+            started_at: @started_at,
+            samples: @samples,
+            weights: [1] * @samples.size,
+            timestamps: @timestamps,
+            sample_categories: [0] * @samples.size,
+          }
+        })
+        result.instance_variable_set(:@meta, {
+          started_at: @started_at
+        })
+        result
+      end
+    end
+
+    class RetainedCollector < Collector
+      def initialize(mode, options)
+        @stack_table = StackTable.new
+        @heap_tracker = HeapTracker.new(@stack_table)
+
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        super
+      end
+
+      def start
+        @heap_tracker.collect
+      end
+
+      def drain
+        @heap_tracker.drain
+      end
+
+      def finish
+        @heap_tracker.drain
+
+        GC.start
+
+        @stack_table.finalize
+
+        GC.start
+
+        @heap_tracker.lock
+        tracker_data = @heap_tracker.data
+
+        samples = tracker_data.fetch(:samples)
+        weights = tracker_data.fetch(:weights)
+
+        result = Result.new
+        result.instance_variable_set(:@threads, {
+          0 => {
+            tid: 0,
+            name: "retained memory",
+            started_at: @started_at,
+            samples: samples,
+            weights: weights,
+            sample_categories: [0] * samples.size,
+          }
+        })
+        result.instance_variable_set(:@meta, {
+          started_at: @started_at
+        })
+        result
+      end
+    end
+
+    def self.new(mode, options = {})
+      return super unless Collector.equal?(self)
+
+      case mode
+      when :wall
+        TimeCollector.new(mode, options)
+      when :custom
+        CustomCollector.new(mode, options)
+      when :retained
+        RetainedCollector.new(mode, options)
+      else
+        raise ArgumentError, "invalid mode: #{mode.inspect}"
+      end
+    end
+
     def initialize(mode, options = {})
       @gc = options.fetch(:gc, true) && (mode == :retained)
       GC.start if @gc
@@ -30,14 +134,20 @@ module Vernier
       @user_metadata = options[:metadata] || {}
     end
 
+    attr_reader :stack_table
+
     private def add_hook(hook)
-      case hook.to_sym
+      case hook.to_s.to_sym
       when :rails, :activesupport
         @hooks << Vernier::Hooks::ActiveSupport.new(self)
       when :memory_usage
         @hooks << Vernier::Hooks::MemoryUsage.new(self)
       else
-        warn "unknown hook: #{hook.inspect}"
+        if hook.respond_to?(:new)
+          @hooks << hook.new(self)
+        else
+          warn "unknown hook: #{hook.inspect}"
+        end
       end
     end
 

data/lib/vernier/heap_tracker.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Vernier
+  # Plan: The heap tracker can be in a few states:
+  #  * Idle
+  #  * Collecting
+  #    * Watching for new objects
+  #    * Watching for freed objects
+  #  * Draining
+  #    * Ignoring new objects
+  #    * Watching for freed objects
+  #  * Locked
+  #    * Ignoring new objects
+  #    * Ignoring freed objects
+  #    * Marking all existing objects (not yet implemented)
+  #    * N.B. This prevents any objects which the tracker has seen from being GC'd
+  class HeapTracker
+    attr_reader :stack_table
+
+    def self.new(stack_table = StackTable.new)
+      _new(stack_table)
+    end
+
+    def inspect
+      "#<#{self.class} allocated_objects=#{allocated_objects} freed_objects=#{freed_objects} stack_table=#{stack_table.inspect}>"
+    end
+
+    def self.track(&block)
+      tracker = new
+      tracker.track(&block)
+      tracker
+    end
+
+    def track
+      collect
+      yield self
+    ensure
+      lock
+    end
+
+    def stack(obj)
+      idx = stack_idx(obj)
+      return nil unless idx
+      stack_table.stack(idx)
+    end
+  end
+end

data/lib/vernier/memory_leak_detector.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Vernier
+  class MemoryLeakDetector
+    def self.start_thread(...)
+      detector = new(...)
+      detector.start_thread
+      detector
+    end
+
+    def initialize(idle_time: 0, collect_time:, drain_time: 0, **collector_options)
+      @idle_time = idle_time
+      @collect_time = collect_time
+      @drain_time = drain_time
+      @collector_options = collector_options
+      @thread = nil
+    end
+
+    def start_thread
+      @thread = Thread.new do
+        sleep @idle_time
+
+        collector = Collector.new(:retained, @collector_options)
+        collector.start
+
+        sleep @collect_time
+
+        collector.drain
+
+        sleep @drain_time
+
+        collector.stop
+      end
+    end
+
+    def result
+      @thread&.value
+    end
+  end
+end

data/lib/vernier/output/firefox.rb

@@ -8,35 +8,59 @@ require_relative "filename_filter"
 module Vernier
   module Output
     # https://profiler.firefox.com/
-    # https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js
+    # https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.ts
     class Firefox
       class Categorizer
+        RAILS_COMPONENTS = %w[ activesupport activemodel activerecord actionview
+                               actionpack activejob actionmailer actioncable
+                               activestorage actionmailbox actiontext railties ]
+
+        AVAILABLE_COLORS = %w[ transparent purple green orange yellow lightblue
+                               blue brown magenta red lightred darkgrey grey ]
+
+        ORDERED_CATEGORIES = %w[ Kernel Rails gem Ruby ] # This is in the order of preference
+
         attr_reader :categories
+
         def initialize
           @categories = []
           @categories_by_name = {}
 
-          add_category(name: "Ruby", color: "grey") do |c|
-            rails_components = %w[ activesupport activemodel activerecord
-              actionview actionpack activejob actionmailer actioncable
-              activestorage actionmailbox actiontext railties ]
+          add_category(name: "Kernel", color: "magenta") do |c|
             c.add_subcategory(
-              name: "Rails",
-              matcher: gem_path(*rails_components)
+              name: "Kernel",
+              matcher: starts_with("<internal")
             )
+          end
+
+          add_category(name: "gem", color: "lightblue") do |c|
             c.add_subcategory(
               name: "gem",
               matcher: starts_with(*Gem.path)
             )
+          end
+
+          add_category(name: "Rails", color: "red") do |c|
+            RAILS_COMPONENTS.each do |subcategory|
+              c.add_subcategory(
+                name: subcategory,
+                matcher: gem_path(subcategory)
+              )
+            end
+          end
+
+          add_category(name: "Ruby", color: "purple") do |c|
             c.add_subcategory(
               name: "stdlib",
               matcher: starts_with(RbConfig::CONFIG["rubylibdir"])
             )
           end
+
           add_category(name: "Idle", color: "transparent")
           add_category(name: "Stalled", color: "transparent")
 
           add_category(name: "GC", color: "red")
+
           add_category(name: "cfunc", color: "yellow", matcher: "<cfunc>")
 
           add_category(name: "Thread", color: "grey")
@@ -69,7 +93,10 @@ module Vernier
 
         class Category
           attr_reader :idx, :name, :color, :matcher, :subcategories
+
           def initialize(idx, name:, color:, matcher: nil)
+            raise ArgumentError, "invalid color: #{color}" if color && AVAILABLE_COLORS.none?(color)
+
             @idx = idx
             @name = name
             @color = color
@@ -315,19 +342,13 @@ module Vernier
             func_implementations[func_idx]
           end
 
-          cfunc_category = @categorizer.get_category("cfunc")
-          ruby_category = @categorizer.get_category("Ruby")
           func_categories, func_subcategories = [], []
           filenames.each do |filename|
-            if filename == "<cfunc>"
-              func_categories << cfunc_category
-              func_subcategories << 0
-            else
-              func_categories << ruby_category
-              subcategory = ruby_category.subcategories.detect {|c| c.matches?(filename) }&.idx || 0
-              func_subcategories << subcategory
-            end
+            category, subcategory = categorize_filename(filename)
+            func_categories << category
+            func_subcategories << subcategory
           end
+
           @frame_categories = @stack_table_hash[:frame_table].fetch(:func).map do |func_idx|
             func_categories[func_idx]
           end
@@ -336,6 +357,32 @@ module Vernier
           end
         end
 
+        def categorize_filename(filename)
+          return cfunc_category_and_subcategory if filename == "<cfunc>"
+
+          category, subcategory = find_category_and_subcategory(filename, Categorizer::ORDERED_CATEGORIES)
+          return category, subcategory if subcategory
+
+          ruby_category_and_subcategory
+        end
+
+        def cfunc_category_and_subcategory
+          [@categorizer.get_category("cfunc"), 0]
+        end
+
+        def ruby_category_and_subcategory
+          [@categorizer.get_category("Ruby"), 0]
+        end
+
+        def find_category_and_subcategory(filename, categories)
+          categories.each do |category_name|
+            category = @categorizer.get_category(category_name)
+            subcategory = category.subcategories.detect {|c| c.matches?(filename) }&.idx
+            return category, subcategory if subcategory
+          end
+          [nil, nil]
+        end
+
         def filter_filenames(filenames)
           filter = FilenameFilter.new
           filenames.map do |filename|

data/lib/vernier/output/top.rb

@@ -3,14 +3,16 @@
 module Vernier
   module Output
     class Top
-      def initialize(profile)
+      def initialize(profile, row_limit)
         @profile = profile
+        @row_limit = row_limit
       end
 
       class Table
-        def initialize(header)
+        def initialize(header, row_limit)
           @header = header
           @rows = []
+          @row_limit = row_limit
           yield self
         end
 
@@ -24,7 +26,7 @@ module Vernier
               row_separator,
               format_row(@header),
               row_separator
-            ] + @rows.map do |row|
+            ] + @rows.first(@row_limit).map do |row|
               format_row(row)
             end + [row_separator]
           ).join("\n")
@@ -70,7 +72,7 @@ module Vernier
           top_by_self[name] += weight
         end
 
-        Table.new %w[Samples % name] do |t|
+        Table.new %w[Samples % name], @row_limit do |t|
           top_by_self.sort_by(&:last).reverse.each do |frame, samples|
             pct = 100.0 * samples / total
             t << [samples.to_s, pct.round(1).to_s, frame]

data/lib/vernier/result.rb

@@ -1,26 +1,18 @@
+# frozen_string_literal: true
+
 module Vernier
   class Result
     attr_accessor :stack_table
     alias _stack_table stack_table
 
-    attr_reader :gc_markers
-
-    attr_accessor :hooks
+    attr_accessor :hooks, :pid, :end_time
 
-    attr_accessor :pid, :end_time
-    attr_accessor :threads
-    attr_accessor :meta
-    attr_accessor :mode
+    attr_reader :meta, :threads, :gc_markers
 
     def main_thread
       threads.values.detect {|x| x[:is_main] }
     end
 
-    # TODO: remove these
-    def weights; threads.values.flat_map { _1[:weights] }; end
-    def samples; threads.values.flat_map { _1[:samples] }; end
-    def sample_categories; threads.values.flat_map { _1[:sample_categories] }; end
-
     # Realtime in nanoseconds since the unix epoch
     def started_at
       started_at_mono_ns = meta[:started_at]
@@ -41,10 +33,17 @@ module Vernier
     def write(out:, format: "firefox")
       case format
       when "cpuprofile"
-        File.binwrite(out, to_cpuprofile)
-      when nil, "firefox"
-        gzip = out.end_with?(".gz")
-        File.binwrite(out, to_firefox(gzip:))
+        if out.respond_to?(:write)
+          out.write(to_cpuprofile)
+        else
+          File.binwrite(out, to_cpuprofile)
+        end
+      when "firefox", nil
+        if out.respond_to?(:write)
+          out.write(to_firefox)
+        else
+          File.binwrite(out, to_firefox(gzip: out.end_with?(".gz")))
+        end
       else
         raise ArgumentError, "unknown format: #{format}"
       end
@@ -55,15 +54,15 @@ module Vernier
     end
 
     def inspect
-      "#<#{self.class} #{elapsed_seconds} seconds, #{threads.count} threads, #{samples.count} samples, #{samples.uniq.size} unique>"
+      "#<#{self.class} #{elapsed_seconds rescue "?"} seconds, #{threads.count} threads, #{total_samples} samples, #{total_unique_samples} unique>"
     end
 
     def each_sample
       return enum_for(__method__) unless block_given?
-      samples.size.times do |sample_idx|
-        weight = weights[sample_idx]
-        stack_idx = samples[sample_idx]
-        yield stack(stack_idx), weight
+      threads.values.each do |thread|
+        thread[:samples].zip(thread[:weights]) do |stack_idx, weight|
+          yield stack(stack_idx), weight
+        end
       end
     end
 
@@ -71,8 +70,24 @@ module Vernier
       stack_table.stack(idx)
     end
 
+    def total_weights
+      threads.values.sum { _1[:weights].sum }
+    end
+
     def total_bytes
-      weights.sum
+      unless meta[:mode] == :retained
+        raise NotImplementedError, "total_bytes is only implemented for retained mode"
+      end
+
+      total_weights
+    end
+
+    def total_samples
+      threads.values.sum { _1[:samples].count }
+    end
+
+    def total_unique_samples
+      threads.values.flat_map { _1[:samples] }.uniq.count
     end
   end
 end

data/lib/vernier/stack_table_helpers.rb

@@ -23,15 +23,21 @@ module Vernier
     end
 
     def backtrace(stack_idx)
-      full_stack(stack_idx).map do |stack_idx|
+      last_filename = nil
+      last_lineno = nil
+      full_stack(stack_idx).reverse.map do |stack_idx|
         frame_idx = stack_frame_idx(stack_idx)
         func_idx = frame_func_idx(frame_idx)
         line = frame_line_no(frame_idx)
-        name = func_name(func_idx);
-        filename = func_filename(func_idx);
+        line = last_lineno if line == 0
+        last_lineno = line
+        name = func_name(func_idx)
+        filename = func_path(func_idx)
+        filename = last_filename if filename.empty?
+        last_filename = filename
 
         "#{filename}:#{line}:in '#{name}'"
-      end
+      end.reverse
     end
 
     def full_stack(stack_idx)
@@ -83,9 +89,14 @@ module Vernier
       def line
         stack_table.frame_line_no(idx)
       end
+      alias lineno line
 
       def to_s
-        "#{func}:#{line}"
+        if (line = self.line) == 0
+          func.to_s
+        else
+          "#{func}:#{line}"
+        end
       end
     end
 
@@ -102,14 +113,16 @@ module Vernier
       end
       alias each_frame each
 
-      def [](n)
-        raise RangeError if n < 0
+      def [](offset)
+        raise RangeError if offset < 0
         stack_idx = idx
-        while n > 0
+        while stack_idx && offset > 0
           stack_idx = stack_table.stack_parent_idx(stack_idx)
-          n -= 1
+          offset -= 1
+        end
+        if stack_idx && offset == 0
+          Frame.new(stack_table, stack_table.stack_frame_idx(stack_idx))
         end
-        Frame.new(stack_table, stack_table.stack_frame_idx(stack_idx))
       end
 
       def leaf_frame_idx
@@ -134,6 +147,7 @@ module Vernier
     end
 
     def stack(idx)
+      raise ArgumentError, "invalid index" unless idx
       Stack.new(self, idx)
     end
   end

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "1.8.0"
+  VERSION = "1.9.0"
 end

data/lib/vernier.rb

@@ -3,6 +3,8 @@
 require_relative "vernier/version"
 require_relative "vernier/collector"
 require_relative "vernier/stack_table"
+require_relative "vernier/heap_tracker"
+require_relative "vernier/memory_leak_detector"
 require_relative "vernier/parsed_profile"
 require_relative "vernier/result"
 require_relative "vernier/hooks"
@@ -63,10 +65,4 @@ module Vernier
   def self.trace_retained(**profile_options, &block)
     profile(**profile_options.merge(mode: :retained), &block)
   end
-
-  class Collector
-    def self.new(mode, options = {})
-      _new(mode, options)
-    end
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vernier
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.0
 platform: ruby
 authors:
 - John Hawthorn
@@ -70,6 +70,7 @@ files:
 - bin/console
 - bin/setup
 - bin/vernier
+- examples/custom_hook.rb
 - examples/fiber_stalls.rb
 - examples/gvl_sleep.rb
 - examples/measure_overhead.rb
@@ -79,20 +80,25 @@ files:
 - examples/threaded_http_requests.rb
 - exe/vernier
 - ext/vernier/extconf.rb
+- ext/vernier/heap_tracker.cc
 - ext/vernier/memory.cc
 - ext/vernier/periodic_thread.hh
 - ext/vernier/ruby_type_names.h
 - ext/vernier/signal_safe_semaphore.hh
+- ext/vernier/stack_table.cc
+- ext/vernier/stack_table.hh
 - ext/vernier/timestamp.hh
 - ext/vernier/vernier.cc
 - ext/vernier/vernier.hh
 - lib/vernier.rb
 - lib/vernier/autorun.rb
 - lib/vernier/collector.rb
+- lib/vernier/heap_tracker.rb
 - lib/vernier/hooks.rb
 - lib/vernier/hooks/active_support.rb
 - lib/vernier/hooks/memory_usage.rb
 - lib/vernier/marker.rb
+- lib/vernier/memory_leak_detector.rb
 - lib/vernier/middleware.rb
 - lib/vernier/output/cpuprofile.rb
 - lib/vernier/output/file_listing.rb
@@ -127,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A next generation CRuby profiler
 test_files: []