vernier 1.8.1 → 1.9.0

data/ext/vernier/vernier.hh

@@ -3,8 +3,15 @@
 
 #include "ruby.h"
 
+// HACK: This isn't public, but the objspace ext uses it
+extern "C" size_t rb_obj_memsize_of(VALUE);
+
+#define sym(name) ID2SYM(rb_intern_const(name))
+
 extern VALUE rb_mVernier;
 
 void Init_memory();
+void Init_stack_table();
+void Init_heap_tracker();
 
 #endif /* VERNIER_H */

data/lib/vernier/collector.rb

@@ -5,6 +5,110 @@ require_relative "thread_names"
 
 module Vernier
   class Collector
+    class CustomCollector < Collector
+      def initialize(mode, options)
+        @stack_table = StackTable.new
+
+        @samples = []
+        @timestamps = []
+
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        super
+      end
+
+      def sample
+        @samples << @stack_table.current_stack
+        @timestamps << Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+      end
+
+      def start
+      end
+
+      def finish
+        result = Result.new
+        result.instance_variable_set(:@threads, {
+          0 => {
+            tid: 0,
+            name: "custom",
+            started_at: @started_at,
+            samples: @samples,
+            weights: [1] * @samples.size,
+            timestamps: @timestamps,
+            sample_categories: [0] * @samples.size,
+          }
+        })
+        result.instance_variable_set(:@meta, {
+          started_at: @started_at
+        })
+        result
+      end
+    end
+
+    class RetainedCollector < Collector
+      def initialize(mode, options)
+        @stack_table = StackTable.new
+        @heap_tracker = HeapTracker.new(@stack_table)
+
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        super
+      end
+
+      def start
+        @heap_tracker.collect
+      end
+
+      def drain
+        @heap_tracker.drain
+      end
+
+      def finish
+        @heap_tracker.drain
+
+        GC.start
+
+        @stack_table.finalize
+
+        GC.start
+
+        @heap_tracker.lock
+        tracker_data = @heap_tracker.data
+
+        samples = tracker_data.fetch(:samples)
+        weights = tracker_data.fetch(:weights)
+
+        result = Result.new
+        result.instance_variable_set(:@threads, {
+          0 => {
+            tid: 0,
+            name: "retained memory",
+            started_at: @started_at,
+            samples: samples,
+            weights: weights,
+            sample_categories: [0] * samples.size,
+          }
+        })
+        result.instance_variable_set(:@meta, {
+          started_at: @started_at
+        })
+        result
+      end
+    end
+
+    def self.new(mode, options = {})
+      return super unless Collector.equal?(self)
+
+      case mode
+      when :wall
+        TimeCollector.new(mode, options)
+      when :custom
+        CustomCollector.new(mode, options)
+      when :retained
+        RetainedCollector.new(mode, options)
+      else
+        raise ArgumentError, "invalid mode: #{mode.inspect}"
+      end
+    end
+
     def initialize(mode, options = {})
       @gc = options.fetch(:gc, true) && (mode == :retained)
       GC.start if @gc
@@ -30,14 +134,20 @@ module Vernier
       @user_metadata = options[:metadata] || {}
     end
 
+    attr_reader :stack_table
+
     private def add_hook(hook)
-      case hook.to_sym
+      case hook.to_s.to_sym
       when :rails, :activesupport
         @hooks << Vernier::Hooks::ActiveSupport.new(self)
       when :memory_usage
         @hooks << Vernier::Hooks::MemoryUsage.new(self)
       else
-        warn "unknown hook: #{hook.inspect}"
+        if hook.respond_to?(:new)
+          @hooks << hook.new(self)
+        else
+          warn "unknown hook: #{hook.inspect}"
+        end
       end
     end
 

data/lib/vernier/heap_tracker.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Vernier
+  # Plan: The heap tracker can be in a few states:
+  #  * Idle
+  #  * Collecting
+  #    * Watching for new objects
+  #    * Watching for freed objects
+  #  * Draining
+  #    * Ignoring new objects
+  #    * Watching for freed objects
+  #  * Locked
+  #    * Ignoring new objects
+  #    * Ignoring freed objects
+  #    * Marking all existing objects (not yet implemented)
+  #    * N.B. This prevents any objects which the tracker has seen from being GC'd
+  class HeapTracker
+    attr_reader :stack_table
+
+    def self.new(stack_table = StackTable.new)
+      _new(stack_table)
+    end
+
+    def inspect
+      "#<#{self.class} allocated_objects=#{allocated_objects} freed_objects=#{freed_objects} stack_table=#{stack_table.inspect}>"
+    end
+
+    def self.track(&block)
+      tracker = new
+      tracker.track(&block)
+      tracker
+    end
+
+    def track
+      collect
+      yield self
+    ensure
+      lock
+    end
+
+    def stack(obj)
+      idx = stack_idx(obj)
+      return nil unless idx
+      stack_table.stack(idx)
+    end
+  end
+end

data/lib/vernier/memory_leak_detector.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Vernier
+  class MemoryLeakDetector
+    def self.start_thread(...)
+      detector = new(...)
+      detector.start_thread
+      detector
+    end
+
+    def initialize(idle_time: 0, collect_time:, drain_time: 0, **collector_options)
+      @idle_time = idle_time
+      @collect_time = collect_time
+      @drain_time = drain_time
+      @collector_options = collector_options
+      @thread = nil
+    end
+
+    def start_thread
+      @thread = Thread.new do
+        sleep @idle_time
+
+        collector = Collector.new(:retained, @collector_options)
+        collector.start
+
+        sleep @collect_time
+
+        collector.drain
+
+        sleep @drain_time
+
+        collector.stop
+      end
+    end
+
+    def result
+      @thread&.value
+    end
+  end
+end

data/lib/vernier/result.rb

@@ -1,26 +1,18 @@
+# frozen_string_literal: true
+
 module Vernier
   class Result
     attr_accessor :stack_table
     alias _stack_table stack_table
 
-    attr_reader :gc_markers
-
-    attr_accessor :hooks
+    attr_accessor :hooks, :pid, :end_time
 
-    attr_accessor :pid, :end_time
-    attr_accessor :threads
-    attr_accessor :meta
-    attr_accessor :mode
+    attr_reader :meta, :threads, :gc_markers
 
     def main_thread
       threads.values.detect {|x| x[:is_main] }
     end
 
-    # TODO: remove these
-    def weights; threads.values.flat_map { _1[:weights] }; end
-    def samples; threads.values.flat_map { _1[:samples] }; end
-    def sample_categories; threads.values.flat_map { _1[:sample_categories] }; end
-
     # Realtime in nanoseconds since the unix epoch
     def started_at
       started_at_mono_ns = meta[:started_at]
@@ -41,10 +33,17 @@ module Vernier
     def write(out:, format: "firefox")
       case format
       when "cpuprofile"
-        File.binwrite(out, to_cpuprofile)
-      when nil, "firefox"
-        gzip = out.end_with?(".gz")
-        File.binwrite(out, to_firefox(gzip:))
+        if out.respond_to?(:write)
+          out.write(to_cpuprofile)
+        else
+          File.binwrite(out, to_cpuprofile)
+        end
+      when "firefox", nil
+        if out.respond_to?(:write)
+          out.write(to_firefox)
+        else
+          File.binwrite(out, to_firefox(gzip: out.end_with?(".gz")))
+        end
       else
         raise ArgumentError, "unknown format: #{format}"
       end
@@ -55,15 +54,15 @@ module Vernier
     end
 
     def inspect
-      "#<#{self.class} #{elapsed_seconds} seconds, #{threads.count} threads, #{samples.count} samples, #{samples.uniq.size} unique>"
+      "#<#{self.class} #{elapsed_seconds rescue "?"} seconds, #{threads.count} threads, #{total_samples} samples, #{total_unique_samples} unique>"
     end
 
     def each_sample
       return enum_for(__method__) unless block_given?
-      samples.size.times do |sample_idx|
-        weight = weights[sample_idx]
-        stack_idx = samples[sample_idx]
-        yield stack(stack_idx), weight
+      threads.values.each do |thread|
+        thread[:samples].zip(thread[:weights]) do |stack_idx, weight|
+          yield stack(stack_idx), weight
+        end
       end
     end
 
@@ -71,8 +70,24 @@ module Vernier
       stack_table.stack(idx)
     end
 
+    def total_weights
+      threads.values.sum { _1[:weights].sum }
+    end
+
     def total_bytes
-      weights.sum
+      unless meta[:mode] == :retained
+        raise NotImplementedError, "total_bytes is only implemented for retained mode"
+      end
+
+      total_weights
+    end
+
+    def total_samples
+      threads.values.sum { _1[:samples].count }
+    end
+
+    def total_unique_samples
+      threads.values.flat_map { _1[:samples] }.uniq.count
     end
   end
 end

data/lib/vernier/stack_table_helpers.rb

@@ -23,15 +23,21 @@ module Vernier
     end
 
     def backtrace(stack_idx)
-      full_stack(stack_idx).map do |stack_idx|
+      last_filename = nil
+      last_lineno = nil
+      full_stack(stack_idx).reverse.map do |stack_idx|
         frame_idx = stack_frame_idx(stack_idx)
         func_idx = frame_func_idx(frame_idx)
         line = frame_line_no(frame_idx)
-        name = func_name(func_idx);
-        filename = func_filename(func_idx);
+        line = last_lineno if line == 0
+        last_lineno = line
+        name = func_name(func_idx)
+        filename = func_path(func_idx)
+        filename = last_filename if filename.empty?
+        last_filename = filename
 
         "#{filename}:#{line}:in '#{name}'"
-      end
+      end.reverse
     end
 
     def full_stack(stack_idx)
@@ -83,9 +89,14 @@ module Vernier
       def line
         stack_table.frame_line_no(idx)
       end
+      alias lineno line
 
       def to_s
-        "#{func}:#{line}"
+        if (line = self.line) == 0
+          func.to_s
+        else
+          "#{func}:#{line}"
+        end
       end
     end
 
@@ -102,14 +113,16 @@ module Vernier
       end
       alias each_frame each
 
-      def [](n)
-        raise RangeError if n < 0
+      def [](offset)
+        raise RangeError if offset < 0
         stack_idx = idx
-        while n > 0
+        while stack_idx && offset > 0
           stack_idx = stack_table.stack_parent_idx(stack_idx)
-          n -= 1
+          offset -= 1
+        end
+        if stack_idx && offset == 0
+          Frame.new(stack_table, stack_table.stack_frame_idx(stack_idx))
         end
-        Frame.new(stack_table, stack_table.stack_frame_idx(stack_idx))
       end
 
       def leaf_frame_idx
@@ -134,6 +147,7 @@ module Vernier
     end
 
     def stack(idx)
+      raise ArgumentError, "invalid index" unless idx
       Stack.new(self, idx)
     end
   end

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "1.8.1"
+  VERSION = "1.9.0"
 end

data/lib/vernier.rb

@@ -3,6 +3,8 @@
 require_relative "vernier/version"
 require_relative "vernier/collector"
 require_relative "vernier/stack_table"
+require_relative "vernier/heap_tracker"
+require_relative "vernier/memory_leak_detector"
 require_relative "vernier/parsed_profile"
 require_relative "vernier/result"
 require_relative "vernier/hooks"
@@ -63,10 +65,4 @@ module Vernier
   def self.trace_retained(**profile_options, &block)
     profile(**profile_options.merge(mode: :retained), &block)
   end
-
-  class Collector
-    def self.new(mode, options = {})
-      _new(mode, options)
-    end
-  end
 end

data/vernier.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "lib/vernier/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "vernier"
+  spec.version = Vernier::VERSION
+  spec.authors = ["John Hawthorn"]
+  spec.email = ["john@hawthorn.email"]
+
+  spec.summary = "A next generation CRuby profiler"
+  spec.description = "Next-generation Ruby 3.2.1+ sampling profiler. Tracks multiple threads, GVL activity, GC pauses, idle time, and more."
+  spec.homepage = "https://github.com/jhawthorn/vernier"
+  spec.license = "MIT"
+
+  unless ENV["IGNORE_REQUIRED_RUBY_VERSION"]
+    spec.required_ruby_version = ">= 3.2.1"
+  end
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions = ["ext/vernier/extconf.rb"]
+
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "gvltest"
+  spec.add_development_dependency "rack"
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vernier
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.9.0
 platform: ruby
 authors:
 - John Hawthorn
@@ -70,6 +70,7 @@ files:
 - bin/console
 - bin/setup
 - bin/vernier
+- examples/custom_hook.rb
 - examples/fiber_stalls.rb
 - examples/gvl_sleep.rb
 - examples/measure_overhead.rb
@@ -79,20 +80,25 @@ files:
 - examples/threaded_http_requests.rb
 - exe/vernier
 - ext/vernier/extconf.rb
+- ext/vernier/heap_tracker.cc
 - ext/vernier/memory.cc
 - ext/vernier/periodic_thread.hh
 - ext/vernier/ruby_type_names.h
 - ext/vernier/signal_safe_semaphore.hh
+- ext/vernier/stack_table.cc
+- ext/vernier/stack_table.hh
 - ext/vernier/timestamp.hh
 - ext/vernier/vernier.cc
 - ext/vernier/vernier.hh
 - lib/vernier.rb
 - lib/vernier/autorun.rb
 - lib/vernier/collector.rb
+- lib/vernier/heap_tracker.rb
 - lib/vernier/hooks.rb
 - lib/vernier/hooks/active_support.rb
 - lib/vernier/hooks/memory_usage.rb
 - lib/vernier/marker.rb
+- lib/vernier/memory_leak_detector.rb
 - lib/vernier/middleware.rb
 - lib/vernier/output/cpuprofile.rb
 - lib/vernier/output/file_listing.rb
@@ -105,6 +111,7 @@ files:
 - lib/vernier/stack_table_helpers.rb
 - lib/vernier/thread_names.rb
 - lib/vernier/version.rb
+- vernier.gemspec
 homepage: https://github.com/jhawthorn/vernier
 licenses:
 - MIT
@@ -126,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A next generation CRuby profiler
 test_files: []