vernier 0.5.1 → 0.7.0

data/lib/vernier/autorun.rb

@@ -17,10 +17,11 @@ module Vernier
 
     def self.start
       interval = options.fetch(:interval, 500).to_i
+      allocation_sample_rate = options.fetch(:allocation_sample_rate, 0).to_i
 
       STDERR.puts("starting profiler with interval #{interval}")
 
-      @collector = Vernier::Collector.new(:wall, interval:)
+      @collector = Vernier::Collector.new(:wall, interval:, allocation_sample_rate:)
       @collector.start
     end
 

data/lib/vernier/collector.rb

@@ -1,12 +1,40 @@
 # frozen_string_literal: true
 
 require_relative "marker"
+require_relative "thread_names"
 
 module Vernier
   class Collector
-    def initialize(mode)
+    def initialize(mode, options = {})
+      if options.fetch(:gc, true) && (mode == :retained)
+        GC.start
+      end
+
       @mode = mode
+      @out = options[:out]
+
       @markers = []
+      @hooks = []
+
+      @thread_names = ThreadNames.new
+
+      if options[:hooks]
+        Array(options[:hooks]).each do |hook|
+          add_hook(hook)
+        end
+      end
+      @hooks.each do |hook|
+        hook.enable
+      end
+    end
+
+    private def add_hook(hook)
+      case hook
+      when :rails, :activesupport
+        @hooks << Vernier::Hooks::ActiveSupport.new(self)
+      else
+        warn "Unknown hook: #{hook}"
+      end
     end
 
     ##
@@ -47,6 +75,19 @@ module Vernier
     def stop
       result = finish
 
+      result.instance_variable_set("@stack_table", stack_table.to_h)
+      @thread_names.finish
+
+      @hooks.each do |hook|
+        hook.disable
+      end
+
+      result.threads.each do |obj_id, thread|
+        thread[:name] ||= @thread_names[obj_id]
+      end
+
+      result.hooks = @hooks
+
       end_time = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
       result.pid = Process.pid
       result.end_time = end_time
@@ -65,6 +106,10 @@ module Vernier
 
       result.instance_variable_set(:@markers, markers)
 
+      if @out
+        result.write(out: @out)
+      end
+
       result
     end
   end

data/lib/vernier/hooks/active_support.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Vernier
+  module Hooks
+    class ActiveSupport
+      FIREFOX_MARKER_SCHEMA = Ractor.make_shareable([
+        {
+          name: "sql.active_record",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "sql", format: "string" },
+            { key: "name", format: "string" },
+            { key: "type_casted_binds", label: "binds", format: "string"
+            }
+          ]
+        },
+        {
+          name: "instantiation.active_record",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "record_count", format: "integer" },
+            { key: "class_name", format: "string" }
+          ]
+        },
+        {
+          name: "process_action.action_controller",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "controller", format: "string" },
+            { key: "action", format: "string" },
+            { key: "status", format: "integer" },
+            { key: "path", format: "string" },
+            { key: "method", format: "string" }
+          ]
+        },
+        {
+          name: "cache_read.active_support",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "key", format: "string" },
+            { key: "store", format: "string" },
+            { key: "hit", format: "string" },
+            { key: "super_operation", format: "string" }
+          ]
+        },
+        {
+          name: "cache_read_multi.active_support",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "key", format: "string" },
+            { key: "store", format: "string" },
+            { key: "hit", format: "string" },
+            { key: "super_operation", format: "string" }
+          ]
+        },
+        {
+          name: "cache_fetch_hit.active_support",
+          display: [ "marker-chart", "marker-table" ],
+          data: [
+            { key: "key", format: "string" },
+            { key: "store", format: "string" }
+          ]
+        }
+      ])
+
+      SERIALIZED_KEYS = FIREFOX_MARKER_SCHEMA.map do |format|
+        [
+          format[:name],
+          format[:data].map { _1[:key].to_sym }.freeze
+        ]
+      end.to_h.freeze
+
+      def initialize(collector)
+        @collector = collector
+      end
+
+      def enable
+        require "active_support"
+        @subscription = ::ActiveSupport::Notifications.monotonic_subscribe(/\A[^!]/) do |name, start, finish, id, payload|
+          # Notifications.publish API may reach here without proper timing information included
+          unless Float === start && Float === finish
+            next
+          end
+
+          data = { type: name }
+          if keys = SERIALIZED_KEYS[name]
+            keys.each do |key|
+              data[key] = payload[key]
+            end
+          end
+          @collector.add_marker(
+            name: name,
+            start: (start * 1_000_000_000.0).to_i,
+            finish: (finish * 1_000_000_000.0).to_i,
+            data: data
+          )
+        end
+      end
+
+      def disable
+        ::ActiveSupport::Notifications.unsubscribe(@subscription)
+        @subscription = nil
+      end
+
+      def firefox_marker_schema
+        FIREFOX_MARKER_SCHEMA
+      end
+    end
+  end
+end

data/lib/vernier/hooks.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Vernier
+  module Hooks
+    autoload :ActiveSupport, "vernier/hooks/active_support"
+  end
+end

data/lib/vernier/middleware.rb

@@ -0,0 +1,32 @@
+module Vernier
+  class Middleware
+    def initialize(app, permit: ->(_) { true })
+      @app = app
+      @permit = permit
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+      return @app.call(env) unless request.GET.has_key?("vernier")
+
+      permitted = @permit.call(request)
+      return @app.call(env) unless permitted
+
+      interval = request.GET.fetch(:vernier_interval, 200).to_i
+      allocation_sample_rate = request.GET.fetch(:vernier_allocation_sample_rate, 200).to_i
+
+      result = Vernier.trace(interval:, allocation_sample_rate:, hooks: [:rails]) do
+        @app.call(env)
+      end
+      body = result.to_gecko
+      filename = "#{request.path.gsub("/", "_")}_#{DateTime.now.strftime("%Y-%m-%d-%H-%M-%S")}.vernier.json"
+      headers = {
+        "Content-Type" => "application/json; charset=utf-8",
+        "Content-Disposition" => "attachment; filename=\"#{filename}\"",
+        "Content-Length" => body.bytesize.to_s
+      }
+
+      Rack::Response.new(body, 200, headers).finish
+    end
+  end
+end

data/lib/vernier/output/firefox.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "rbconfig"
 
 module Vernier
   module Output
@@ -89,7 +90,7 @@ module Vernier
       end
 
       def output
-        ::JSON.generate(data)
+        ::JSON.fast_generate(data)
       end
 
       private
@@ -133,7 +134,8 @@ module Vernier
                 color: category.color,
                 subcategories: []
               }
-            end
+            end,
+            sourceCodeIsNotOnSearchfox: true
           },
           libs: [],
           threads: thread_data
@@ -141,9 +143,15 @@ module Vernier
       end
 
       def marker_schema
+        hook_additions = profile.hooks.flat_map do |hook|
+          if hook.respond_to?(:firefox_marker_schema)
+            hook.firefox_marker_schema
+          end
+        end.compact
+
         [
           {
-            name: "GVL_THREAD_RESUMED",
+            name: "THREAD_RUNNING",
             display: [ "marker-chart", "marker-table" ],
             data: [
               {
@@ -151,19 +159,57 @@ module Vernier
                 value: "The thread has acquired the GVL and is executing"
               }
             ]
-          }
+          },
+          {
+            name: "THREAD_STALLED",
+            display: [ "marker-chart", "marker-table" ],
+            data: [
+              {
+                label: "Description",
+                value: "The thread is ready, but stalled waiting for the GVL to be available"
+              }
+            ]
+          },
+          {
+            name: "THREAD_SUSPENDED",
+            display: [ "marker-chart", "marker-table" ],
+            data: [
+              {
+                label: "Description",
+                value: "The thread has voluntarily released the GVL (ex. to sleep, for I/O, waiting on a lock)"
+              }
+            ]
+          },
+          {
+            name: "GC_PAUSE",
+            display: [ "marker-chart", "marker-table", "timeline-overview" ],
+            tooltipLabel: "{marker.name} - {marker.data.state}",
+            data: [
+              {
+                label: "Description",
+                value: "All threads are paused as GC is performed"
+              }
+            ]
+          },
+          *hook_additions
         ]
       end
 
       class Thread
         attr_reader :profile
 
-        def initialize(ruby_thread_id, profile, categorizer, name:, tid:, samples:, weights:, timestamps: nil, sample_categories: nil, markers:, started_at:, stopped_at: nil)
+        def initialize(ruby_thread_id, profile, categorizer, name:, tid:, samples:, weights:, timestamps: nil, sample_categories: nil, markers:, started_at:, stopped_at: nil, allocations: nil, is_main: nil)
           @ruby_thread_id = ruby_thread_id
           @profile = profile
           @categorizer = categorizer
           @tid = tid
-          @name = pretty_name(name)
+          @allocations = allocations
+          @name = name
+          @is_main = is_main
+          if is_main.nil?
+            @is_main = @ruby_thread_id == ::Thread.main.object_id
+          end
+          @is_main = true if profile.threads.size == 1
 
           timestamps ||= [0] * samples.size
           @samples, @weights, @timestamps = samples, weights, timestamps
@@ -184,41 +230,67 @@ module Vernier
           @func_names = names.map do |name|
             @strings[name]
           end
-          @filenames = filenames.map do |filename|
+
+          @filenames = filter_filenames(filenames).map do |filename|
             @strings[filename]
           end
 
           lines = profile.frame_table.fetch(:line)
 
-          @frame_implementations = filenames.zip(lines).map do |filename, line|
+          func_implementations = filenames.map do |filename|
             # Must match strings in `src/profile-logic/profile-data.js`
             # inside the firefox profiler. See `getFriendlyStackTypeName`
             if filename == "<cfunc>"
               @strings["native"]
             else
-              # FIXME: We need to get upstream support for JIT frames
-              if line == -1
-                @strings["yjit"]
-              else
-                # nil means interpreter
-                nil
-              end
+              # nil means interpreter
+              nil
             end
           end
+          @frame_implementations = profile.frame_table.fetch(:func).map do |func_idx|
+            func_implementations[func_idx]
+          end
 
-          @frame_categories = filenames.map do |filename|
+          func_categories = filenames.map do |filename|
             @categorizer.categorize(filename)
           end
+          @frame_categories = profile.frame_table.fetch(:func).map do |func_idx|
+            func_categories[func_idx]
+          end
+        end
+
+        def filter_filenames(filenames)
+          pwd = "#{Dir.pwd}/"
+          gem_regex = %r{\A#{Regexp.union(Gem.path)}/gems/}
+          gem_match_regex = %r{\A#{Regexp.union(Gem.path)}/gems/([a-zA-Z](?:[a-zA-Z0-9\.\_]|-[a-zA-Z])*)-([0-9][0-9A-Za-z\-_\.]*)/(.*)\z}
+          rubylibdir = "#{RbConfig::CONFIG["rubylibdir"]}/"
+
+          filenames.map do |filename|
+            if filename.match?(gem_regex)
+              gem_match_regex =~ filename
+              "gem:#$1-#$2:#$3"
+            elsif filename.start_with?(pwd)
+              filename.delete_prefix(pwd)
+            elsif filename.start_with?(rubylibdir)
+              path = filename.delete_prefix(rubylibdir)
+              "rubylib:#{RUBY_VERSION}:#{path}"
+            else
+              filename
+            end
+          end
         end
 
         def data
+          started_at = (@started_at - 0) / 1_000_000.0
+          stopped_at = (@stopped_at - 0) / 1_000_000.0 if @stopped_at
+
           {
             name: @name,
-            isMainThread: @ruby_thread_id == ::Thread.main.object_id || (profile.threads.size == 1),
-            processStartupTime: 0, # FIXME
-            processShutdownTime: nil, # FIXME
-            registerTime: (@started_at - 0) / 1_000_000.0,
-            unregisterTime: ((@stopped_at - 0) / 1_000_000.0 if @stopped_at),
+            isMainThread: @is_main,
+            processStartupTime: started_at,
+            processShutdownTime: stopped_at,
+            registerTime: started_at,
+            unregisterTime: stopped_at,
             pausedRanges: [],
             pid: profile.pid || Process.pid,
             tid: @tid,
@@ -226,6 +298,7 @@ module Vernier
             funcTable: func_table,
             nativeSymbols: {},
             samples: samples_table,
+            jsAllocations: allocations_table,
             stackTable: stack_table,
             resourceTable: {
               length: 0,
@@ -236,7 +309,7 @@ module Vernier
             },
             markers: markers_table,
             stringArray: string_table
-          }
+          }.compact
         end
 
         def markers_table
@@ -255,12 +328,14 @@ module Vernier
             end_times << (finish&./(1_000_000.0))
             phases << phase
 
-            category = case name
-            when /\AGC/ then gc_category.idx
-            when /\AThread/ then thread_category.idx
-            else
-              0
-            end
+            category =
+              if name.start_with?("GC")
+                gc_category.idx
+              elsif name.start_with?("Thread")
+                thread_category.idx
+              else
+                0
+              end
 
             categories << category
             data << datum
@@ -277,6 +352,25 @@ module Vernier
           }
         end
 
+        def allocations_table
+          return nil if !@allocations
+          samples, weights, timestamps = @allocations.values_at(:samples, :weights, :timestamps)
+          return nil if samples.size == 0
+          size = samples.size
+          timestamps = timestamps.map { _1 / 1_000_000.0 }
+          ret = {
+            "time": timestamps,
+            "className": ["Object"]*size,
+            "typeName": ["JSObject"]*size,
+            "coarseType": ["Object"]*size,
+            "weight": weights,
+            "inNursery": [false] * size,
+            "stack": samples,
+            "length": size
+          }
+          ret
+        end
+
         def samples_table
           samples = @samples
           weights = @weights
@@ -366,14 +460,21 @@ module Vernier
 
           cfunc_idx = @strings["<cfunc>"]
           is_js = @filenames.map { |fn| fn != cfunc_idx }
+          line_numbers = profile.func_table.fetch(:first_line).map.with_index do |line, i|
+            if is_js[i] || line != 0
+              line
+            else
+              nil
+            end
+          end
           {
             name: @func_names,
             isJS: is_js,
             relevantForJS: is_js,
             resource: [-1] * size, # set to unidentified for now
             fileName: @filenames,
-            lineNumber: profile.func_table.fetch(:first_line),
-            columnNumber: [0] * size,
+            lineNumber: line_numbers,
+            columnNumber: [nil] * size,
             #columnNumber: functions.map { _1.column },
             length: size
           }
@@ -385,25 +486,6 @@ module Vernier
 
         private
 
-        def pretty_name(name)
-          if name.empty?
-            begin
-              tr = ObjectSpace._id2ref(@ruby_thread_id)
-              name = tr.inspect if tr
-            rescue RangeError
-              # Thread was already GC'd
-            end
-          end
-          return name unless name.start_with?("#<Thread")
-          pretty = []
-          obj_address = name[/Thread:(0x\w+)/,1]
-          best_id = name[/\#<Thread:0x\w+@?\s?(.*)\s+\S+>/,1] || ""
-          Gem.path.each { |gem_dir| best_id = best_id.gsub(gem_dir, "...") }
-          pretty << best_id unless best_id.empty?
-          pretty << "(#{obj_address})"
-          pretty.join(' ')
-        end
-
         def gc_category
           @categorizer.get_category("GC")
         end

data/lib/vernier/result.rb

@@ -1,12 +1,29 @@
 module Vernier
   class Result
-    attr_reader :stack_table, :frame_table, :func_table
+    def stack_table
+      @stack_table[:stack_table]
+    end
+
+    def frame_table
+      @stack_table[:frame_table]
+    end
+
+    def func_table
+      @stack_table[:func_table]
+    end
+
     attr_reader :markers
 
+    attr_accessor :hooks
+
     attr_accessor :pid, :end_time
     attr_accessor :threads
     attr_accessor :meta
 
+    def main_thread
+      threads.values.detect {|x| x[:is_main] }
+    end
+
     # TODO: remove these
     def weights; threads.values.flat_map { _1[:weights] }; end
     def samples; threads.values.flat_map { _1[:samples] }; end

data/lib/vernier/stack_table.rb

@@ -0,0 +1,42 @@
+module Vernier
+  class StackTable
+    def to_h
+      {
+        stack_table: {
+          parent: stack_count.times.map { stack_parent_idx(_1) },
+          frame:  stack_count.times.map { stack_frame_idx(_1) }
+        },
+        frame_table: {
+          func: frame_count.times.map { frame_func_idx(_1) },
+          line:  frame_count.times.map { frame_line_no(_1) }
+        },
+        func_table: {
+          name: func_count.times.map { func_name(_1) },
+          filename: func_count.times.map { func_filename(_1) },
+          first_line: func_count.times.map { func_first_lineno(_1) }
+        }
+      }
+    end
+
+    def backtrace(stack_idx)
+      full_stack(stack_idx).map do |stack_idx|
+        frame_idx = stack_frame_idx(stack_idx)
+        func_idx = frame_func_idx(frame_idx)
+        line = frame_line_no(frame_idx)
+        name = func_name(func_idx);
+        filename = func_filename(func_idx);
+
+        "#{filename}:#{line}:in '#{name}'"
+      end
+    end
+
+    def full_stack(stack_idx)
+      full_stack = []
+      while stack_idx
+        full_stack << stack_idx
+        stack_idx = stack_parent_idx(stack_idx)
+      end
+      full_stack
+    end
+  end
+end

data/lib/vernier/thread_names.rb

@@ -0,0 +1,52 @@
+module Vernier
+  # Collects names of all seen threads
+  class ThreadNames
+    def initialize
+      @names = {}
+      @tp = TracePoint.new(:thread_end) do |e|
+        collect_thread(e.self)
+      end
+      @tp.enable
+    end
+
+    def [](object_id)
+      @names[object_id] || "unknown thread #{object_id}"
+    end
+
+    def finish
+      collect_running
+      @tp.disable
+    end
+
+    private
+
+    def collect_running
+      Thread.list.each do |th|
+        collect_thread(th)
+      end
+    end
+
+    def collect_thread(th)
+      @names[th.object_id] = pretty_name(th)
+    end
+
+    def pretty_name(thread)
+      name = thread.name
+      return name if name && !name.empty?
+
+      if thread == Thread.main
+        return "main"
+      end
+
+      name = Thread.instance_method(:inspect).bind_call(thread)
+      pretty = []
+      best_id = name[/\#<Thread:0x\w+@?\s?(.*)\s+\S+>/, 1]
+      if best_id
+        Gem.path.each { |gem_dir| best_id.gsub!(gem_dir, "") }
+        pretty << best_id unless best_id.empty?
+      end
+      pretty << "(#{thread.object_id})"
+      pretty.join(' ')
+    end
+  end
+end

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "0.5.1"
+  VERSION = "0.7.0"
 end