vernier 1.4.0 → 1.6.0

data/lib/vernier/autorun.rb

@@ -29,8 +29,24 @@ module Vernier
     def self.stop
       result = @collector.stop
       @collector = nil
+
       output_path = options[:output]
-      output_path ||= Tempfile.create(["profile", ".vernier.json.gz"]).path
+      unless output_path
+        output_dir = options[:output_dir]
+        unless output_dir
+          if File.writable?(".")
+            output_dir = "."
+          else
+            output_dir = Dir.tmpdir
+          end
+        end
+        prefix = "profile-"
+        timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
+        suffix = ".vernier.json.gz"
+
+        output_path = File.expand_path("#{output_dir}/#{prefix}#{timestamp}-#{$$}#{suffix}")
+      end
+
       result.write(out: output_path)
 
       STDERR.puts(result.inspect)

data/lib/vernier/collector.rb

@@ -31,6 +31,8 @@ module Vernier
       case hook.to_sym
       when :rails, :activesupport
         @hooks << Vernier::Hooks::ActiveSupport.new(self)
+      when :memory_usage
+        @hooks << Vernier::Hooks::MemoryUsage.new(self)
       else
         warn "unknown hook: #{hook.inspect}"
       end
@@ -97,18 +99,44 @@ module Vernier
 
       marker_strings = Marker.name_table
 
-      markers = self.markers.map do |(tid, type, phase, ts, te, stack, extra_info)|
-        name = marker_strings[type]
-        sym = Marker::MARKER_SYMBOLS[type]
-        data = { type: sym }
-        data[:cause] = { stack: stack } if stack
-        data.merge!(extra_info) if extra_info
-        [tid, name, ts, te, phase, data]
+      markers_by_thread_id = (@markers || []).group_by(&:first)
+
+      result.threads.each do |tid, thread|
+        last_fiber = nil
+        markers = []
+
+        markers.concat markers_by_thread_id.fetch(tid, [])
+
+        original_markers = thread[:markers] || []
+        original_markers += result.gc_markers || []
+        original_markers.each do |data|
+          type, phase, ts, te, stack, extra_info = data
+          if type == Marker::Type::FIBER_SWITCH
+            if last_fiber
+              start_event = markers[last_fiber]
+              markers << [nil, "Fiber Running", start_event[2], ts, Marker::Phase::INTERVAL, start_event[5].merge(type: "Fiber Running", cause: nil)]
+            end
+            last_fiber = markers.size
+          end
+          name = marker_strings[type]
+          sym = Marker::MARKER_SYMBOLS[type]
+          data = { type: sym }
+          data[:cause] = { stack: stack } if stack
+          data.merge!(extra_info) if extra_info
+          markers << [tid, name, ts, te, phase, data]
+        end
+        if last_fiber
+          end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+          start_event = markers[last_fiber]
+          markers << [nil, "Fiber Running", start_event[2], end_time, Marker::Phase::INTERVAL, start_event[5].merge(type: "Fiber Running", cause: nil)]
+        end
+
+        thread[:markers] = markers
       end
 
-      markers.concat @markers
+      #markers.concat @markers
 
-      result.instance_variable_set(:@markers, markers)
+      #result.instance_variable_set(:@markers, markers)
 
       if @out
         result.write(out: @out)

data/lib/vernier/hooks/memory_usage.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Vernier
+  module Hooks
+    class MemoryUsage
+      def initialize(collector)
+        @collector = collector
+        @tracker = Vernier::MemoryTracker.new
+      end
+
+      def enable
+        @tracker.start
+      end
+
+      def disable
+        @tracker.stop
+      end
+
+      def firefox_counters
+        timestamps, memory = @tracker.results
+        memory = ([0] + memory).each_cons(2).map { _2 - _1 }
+        {
+          name: "memory",
+          category: "Memory",
+          description: "Memory usage in bytes",
+          pid: Process.pid,
+          mainThreadIndex: 0,
+          samples: {
+            time: timestamps.map { _1 / 1_000_000.0 },
+            count: memory,
+            length: timestamps.length
+          }
+        }
+      end
+    end
+  end
+end

data/lib/vernier/hooks.rb

@@ -3,5 +3,6 @@
 module Vernier
   module Hooks
     autoload :ActiveSupport, "vernier/hooks/active_support"
+    autoload :MemoryUsage, "vernier/hooks/memory_usage"
   end
 end

data/lib/vernier/marker.rb

@@ -26,6 +26,8 @@ module Vernier
     MARKER_STRINGS[Type::THREAD_STALLED] = "Thread Stalled"
     MARKER_STRINGS[Type::THREAD_SUSPENDED] = "Thread Suspended"
 
+    MARKER_STRINGS[Type::FIBER_SWITCH] = "Fiber Switch"
+
     MARKER_STRINGS.freeze
 
     ##

data/lib/vernier/middleware.rb

@@ -1,6 +1,6 @@
 module Vernier
   class Middleware
-    def initialize(app, permit: ->(_) { true })
+    def initialize(app, permit: ->(_env) { true })
       @app = app
       @permit = permit
     end

data/lib/vernier/output/file_listing.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative "filename_filter"
+require "cgi/util"
+
+module Vernier
+  module Output
+    class FileListing
+      class SamplesByLocation
+        attr_accessor :self, :total
+        def initialize
+          @self = @total = 0
+        end
+
+        def +(other)
+          ret = SamplesByLocation.new
+          ret.self = @self + other.self
+          ret.total = @total + other.total
+          ret
+        end
+      end
+
+      def initialize(profile)
+        @profile = profile
+      end
+
+      def samples_by_file
+        thread = @profile.main_thread
+        if Hash === thread
+          # live profile
+          stack_table = @profile._stack_table
+          filename_filter = FilenameFilter.new
+        else
+          stack_table = thread.stack_table
+          filename_filter = ->(x) { x }
+        end
+
+        weights = thread[:weights]
+        samples = thread[:samples]
+
+        self_samples_by_frame = Hash.new do |h, k|
+          h[k] = SamplesByLocation.new
+        end
+
+        samples.zip(weights).each do |stack_idx, weight|
+          # self time
+          top_frame_index = stack_table.stack_frame_idx(stack_idx)
+          self_samples_by_frame[top_frame_index].self += weight
+
+          # total time
+          while stack_idx
+            frame_idx = stack_table.stack_frame_idx(stack_idx)
+            self_samples_by_frame[frame_idx].total += weight
+            stack_idx = stack_table.stack_parent_idx(stack_idx)
+          end
+        end
+
+        samples_by_file = Hash.new do |h, k|
+          h[k] = Hash.new do |h2, k2|
+            h2[k2] = SamplesByLocation.new
+          end
+        end
+
+        self_samples_by_frame.each do |frame, samples|
+          line = stack_table.frame_line_no(frame)
+          func_index = stack_table.frame_func_idx(frame)
+          filename = stack_table.func_filename(func_index)
+
+          samples_by_file[filename][line] += samples
+        end
+
+        samples_by_file.transform_keys! do |filename|
+          filename_filter.call(filename)
+        end
+      end
+
+      def output(template: nil)
+        output = +""
+
+        relevant_files = samples_by_file.select do |k, v|
+          next if k.start_with?("gem:")
+          next if k.start_with?("rubylib:")
+          next if k.start_with?("<")
+          v.values.map(&:total).sum > total * 0.01
+        end
+
+        if template == "html"
+          html_output(output, relevant_files)
+        else
+          relevant_files.keys.sort.each do |filename|
+            output << "="*80 << "\n"
+            output << filename << "\n"
+            output << "-"*80 << "\n"
+            format_file(output, filename, samples_by_file, total: total)
+          end
+          output << "="*80 << "\n"
+        end
+      end
+
+      def total
+        thread = @profile.main_thread
+        thread[:weights].sum
+      end
+
+      def format_file(output, filename, all_samples, total:)
+        samples = all_samples[filename]
+
+        # file_name, lines, file_wall, file_cpu, file_idle, file_sort
+        output << sprintf(" TOTAL |  SELF  | LINE SOURCE\n")
+        File.readlines(filename).each_with_index do |line, i|
+          lineno = i + 1
+          calls = samples[lineno]
+
+          if calls && calls.total > 0
+            output << sprintf("%5.1f%% | %5.1f%% | % 4i  %s", 100 * calls.total / total.to_f, 100 * calls.self / total.to_f, lineno, line)
+          else
+            output << sprintf("       |        | % 4i  %s", lineno, line)
+          end
+        end
+      end
+
+      def html_output(output, relevant_files)
+        output << "<pre>"
+        output << "  SELF     FILE\n"
+        relevant_files.sort_by {|k, v| -v.values.map(&:self).sum }.each do |filename, file_contents|
+          tmpl = "<details style=\"display:inline-block;vertical-align:top;\"><summary>%s</summary>"
+          output << sprintf("% 5.1f%%   #{tmpl}\n", file_contents.values.map(&:self).sum * 100 / total.to_f, filename)
+          format_file_html(output, filename, relevant_files)
+          output << "</details>\n"
+        end
+        output << "</pre>"
+      end
+
+      def format_file_html(output, filename, relevant_files)
+        samples = relevant_files[filename]
+
+        # file_name, lines, file_wall, file_cpu, file_idle, file_sort
+        output << sprintf(" TOTAL |  SELF  | LINE SOURCE\n")
+        File.readlines(filename).each_with_index do |line, i|
+          lineno = i + 1
+          calls = samples[lineno]
+
+          if calls && calls.total > 0
+            output << sprintf("%5.1f%% | %5.1f%% | % 4i  %s", 100 * calls.total / total.to_f, 100 * calls.self / total.to_f, lineno, CGI::escapeHTML(line))
+          else
+            output << sprintf("       |        | % 4i  %s", lineno, CGI::escapeHTML(line))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vernier/output/filename_filter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Vernier
+  module Output
+    class FilenameFilter
+      def initialize
+        @pwd = "#{Dir.pwd}/"
+        @gem_regex = %r{\A#{Regexp.union(Gem.path)}/gems/}
+        @gem_match_regex = %r{\A#{Regexp.union(Gem.path)}/gems/([a-zA-Z](?:[a-zA-Z0-9\.\_]|-[a-zA-Z])*)-([0-9][0-9A-Za-z\-_\.]*)/(.*)\z}
+        @rubylibdir = "#{RbConfig::CONFIG["rubylibdir"]}/"
+      end
+
+      attr_reader :pwd, :gem_regex, :gem_match_regex, :rubylibdir
+
+      def call(filename)
+        if filename.match?(gem_regex)
+          gem_match_regex =~ filename
+          "gem:#$1-#$2:#$3"
+        elsif filename.start_with?(pwd)
+          filename.delete_prefix(pwd)
+        elsif filename.start_with?(rubylibdir)
+          path = filename.delete_prefix(rubylibdir)
+          "rubylib:#{RUBY_VERSION}:#{path}"
+        else
+          filename
+        end
+      end
+    end
+  end
+end

data/lib/vernier/output/firefox.rb

@@ -3,6 +3,8 @@
 require "json"
 require "rbconfig"
 
+require_relative "filename_filter"
+
 module Vernier
   module Output
     # https://profiler.firefox.com/
@@ -104,15 +106,15 @@ module Vernier
       attr_reader :profile
 
       def data
-        markers_by_thread = profile.markers.group_by { |marker| marker[0] }
+        #markers_by_thread = profile.markers.group_by { |marker| marker[0] }
 
         threads = profile.threads.map do |ruby_thread_id, thread_info|
-          markers = markers_by_thread[ruby_thread_id] || []
+          #markers = markers_by_thread[ruby_thread_id] || []
           Thread.new(
             ruby_thread_id,
             profile,
             @categorizer,
-            markers: markers,
+            #markers: markers,
             **thread_info,
           )
         end
@@ -126,7 +128,7 @@ module Vernier
             product: "Ruby/Vernier",
             stackwalk: 1,
             version: 28,
-            preprocessedProfileVersion: 47,
+            preprocessedProfileVersion: 48,
             symbolicated: true,
             markerSchema: marker_schema,
             sampleUnits: {
@@ -145,11 +147,20 @@ module Vernier
             initialVisibleThreads: threads.each_index.to_a,
             initialSelectedThreads: Array(threads.find_index(&:is_start))
           },
+          counters: counter_data,
           libs: [],
           threads: threads.map(&:data)
         }
       end
 
+      def counter_data
+        profile.hooks.flat_map do |hook|
+          if hook.respond_to?(:firefox_counters)
+            hook.firefox_counters
+          end
+        end.compact
+      end
+
       def marker_schema
         hook_additions = profile.hooks.flat_map do |hook|
           if hook.respond_to?(:firefox_marker_schema)
@@ -201,6 +212,18 @@ module Vernier
               { key: "gc_by", format: "string" },
             ]
           },
+          {
+            name: "FIBER_SWITCH",
+            display: [ "marker-chart", "marker-table", "timeline-overview" ],
+            tooltipLabel: "{marker.name} - {marker.data.fiber_id}",
+            data: [
+              {
+                label: "Description",
+                value: "Switch running Fiber"
+              },
+              { key: "fiber_id", format: "integer" },
+            ]
+          },
           *hook_additions
         ]
       end
@@ -303,23 +326,9 @@ module Vernier
         end
 
         def filter_filenames(filenames)
-          pwd = "#{Dir.pwd}/"
-          gem_regex = %r{\A#{Regexp.union(Gem.path)}/gems/}
-          gem_match_regex = %r{\A#{Regexp.union(Gem.path)}/gems/([a-zA-Z](?:[a-zA-Z0-9\.\_]|-[a-zA-Z])*)-([0-9][0-9A-Za-z\-_\.]*)/(.*)\z}
-          rubylibdir = "#{RbConfig::CONFIG["rubylibdir"]}/"
-
+          filter = FilenameFilter.new
           filenames.map do |filename|
-            if filename.match?(gem_regex)
-              gem_match_regex =~ filename
-              "gem:#$1-#$2:#$3"
-            elsif filename.start_with?(pwd)
-              filename.delete_prefix(pwd)
-            elsif filename.start_with?(rubylibdir)
-              path = filename.delete_prefix(rubylibdir)
-              "rubylib:#{RUBY_VERSION}:#{path}"
-            else
-              filename
-            end
+            filter.call(filename)
           end
         end
 
@@ -478,17 +487,21 @@ module Vernier
         def frame_table
           funcs = @stack_table_hash[:frame_table].fetch(:func)
           lines = @stack_table_hash[:frame_table].fetch(:line)
-          size = funcs.length
+          raise unless lines.size == funcs.size
+
+          size = funcs.size
           none = [nil] * size
-          categories = @frame_categories.map(&:idx)
+          default = [0] * size
+          unidentified = [-1] * size
 
-          raise unless lines.size == funcs.size
+          categories = @frame_categories.map(&:idx)
+          subcategories = @frame_subcategories
 
           {
-            address: [-1] * size,
-            inlineDepth: [0] * size,
+            address: unidentified,
+            inlineDepth: default,
             category: categories,
-            subcategory: nil,
+            subcategory: subcategories,
             func: funcs,
             nativeSymbol: none,
             innerWindowID: none,

data/lib/vernier/output/top.rb

@@ -7,23 +7,75 @@ module Vernier
         @profile = profile
       end
 
+      class Table
+        def initialize(header)
+          @header = header
+          @rows = []
+          yield self
+        end
+
+        def <<(row)
+          @rows << row
+        end
+
+        def to_s
+          (
+            [
+              row_separator,
+              format_row(@header),
+              row_separator
+            ] + @rows.map do |row|
+              format_row(row)
+            end + [row_separator]
+          ).join("\n")
+        end
+
+        def widths
+          @widths ||=
+            (@rows + [@header]).transpose.map do |col|
+              col.map(&:size).max
+            end
+        end
+
+        def row_separator
+          @row_separator = "+" + widths.map { |i| "-" * (i + 2) }.join("+") + "+"
+        end
+
+        def format_row(row)
+          "|" + row.map.with_index { |str, i| " " + str.ljust(widths[i] + 1) }.join("|") + "|"
+        end
+      end
+
       def output
+        thread = @profile.main_thread
+        stack_table =
+          if thread.respond_to?(:stack_table)
+            thread.stack_table
+          else
+            @profile._stack_table
+          end
+
         stack_weights = Hash.new(0)
-        @profile.samples.zip(@profile.weights) do |stack_idx, weight|
+        thread[:samples].zip(thread[:weights]) do |stack_idx, weight|
           stack_weights[stack_idx] += weight
         end
 
+        total = stack_weights.values.sum
+
         top_by_self = Hash.new(0)
         stack_weights.each do |stack_idx, weight|
-          stack = @profile.stack(stack_idx)
-          top_by_self[stack.leaf_frame.name] += weight
+          frame_idx = stack_table.stack_frame_idx(stack_idx)
+          func_idx = stack_table.frame_func_idx(frame_idx)
+          name = stack_table.func_name(func_idx)
+          top_by_self[name] += weight
         end
 
-        s = +""
-        top_by_self.sort_by(&:last).reverse.each do |frame, samples|
-          s << "#{samples}\t#{frame}\n"
-        end
-        s
+        Table.new %w[Samples % name] do |t|
+          top_by_self.sort_by(&:last).reverse.each do |frame, samples|
+            pct = 100.0 * samples / total
+            t << [samples.to_s, pct.round(1).to_s, frame]
+          end
+        end.to_s
       end
     end
   end

data/lib/vernier/parsed_profile.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "stack_table_helpers"
+
+module Vernier
+  class ParsedProfile
+    def self.read_file(filename)
+      # Print the inverted tree from a Vernier profile
+      is_gzip = File.binread(filename, 2) == "\x1F\x8B".b # check for gzip header
+
+      json = if is_gzip
+        require "zlib"
+        Zlib::GzipReader.open(filename) { |gz| gz.read }
+      else
+        File.read filename
+      end
+
+      info = JSON.load json
+
+      new(info)
+    end
+
+    class StackTable
+      def initialize(thread_data)
+        @stack_parents = thread_data["stackTable"]["prefix"]
+        @stack_frames  = thread_data["stackTable"]["frame"]
+        @frame_funcs   = thread_data["frameTable"]["func"]
+        @frame_lines   = thread_data["frameTable"]["line"]
+        @func_names    = thread_data["funcTable"]["name"]
+        @func_filenames = thread_data["funcTable"]["fileName"]
+        #@func_first_linenos = thread_data["funcTable"]["first"]
+        @strings  = thread_data["stringArray"]
+      end
+
+      attr_reader :strings
+
+      def stack_count = @stack_parents.length
+      def frame_count = @frame_funcs.length
+      def func_count = @func_names.length
+
+      def stack_parent_idx(idx) = @stack_parents[idx]
+      def stack_frame_idx(idx) = @stack_frames[idx]
+
+      def frame_func_idx(idx) = @frame_funcs[idx]
+      def frame_line_no(idx) = @frame_lines[idx]
+
+      def func_name_idx(idx) = @func_names[idx]
+      def func_filename_idx(idx) = @func_filenames[idx]
+      def func_name(idx) = @strings[func_name_idx(idx)]
+      def func_filename(idx) = @strings[func_filename_idx(idx)]
+      def func_first_lineno(idx) = @func_first_lineno[idx]
+
+      include StackTableHelpers
+    end
+
+    class Thread
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def stack_table
+        @stack_table ||= StackTable.new(@data)
+      end
+
+      def main_thread?
+        @data["isMainThread"]
+      end
+
+      def samples
+        @data["samples"]["stack"]
+      end
+
+      def weights
+        @data["samples"]["weight"]
+      end
+
+      # Emulate hash
+      def [](name)
+        send(name)
+      end
+    end
+
+    attr_reader :data
+    def initialize(data)
+      @data = data
+    end
+
+    def threads
+      @threads ||=
+        @data["threads"].map do |thread_data|
+          Thread.new(thread_data)
+        end
+    end
+
+    def main_thread
+      threads.detect(&:main_thread?)
+    end
+  end
+end