vernier 0.1.999 → 0.2.0

data/lib/vernier/collector.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "marker"
+
+module Vernier
+  class Collector
+    def initialize(mode)
+      @mode = mode
+    end
+
+    def stop
+      result = finish
+
+      markers = []
+      marker_list = self.markers
+      size = marker_list.size
+      marker_strings = Marker.name_table
+
+      marker_list.each_with_index do |(tid, id, ts), i|
+        name = marker_strings[id]
+        finish = nil
+        phase = Marker::Phase::INSTANT
+
+        if id == Marker::Type::GC_EXIT
+          # skip because these are incorporated in "GC enter"
+        else
+          if id == Marker::Type::GC_ENTER
+            j = i + 1
+
+            name = "GC pause"
+            phase = Marker::Phase::INTERVAL
+
+            while j < size
+              if marker_list[j][1] == Marker::Type::GC_EXIT
+                finish = marker_list[j][2]
+                break
+              end
+
+              j += 1
+            end
+          end
+
+          markers << [tid, name, ts, finish, phase]
+        end
+      end
+
+      result.instance_variable_set(:@markers, markers)
+
+      result
+    end
+  end
+end

data/lib/vernier/marker.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "vernier" # Make sure constants are loaded
+
+module Vernier
+  module Marker
+    # These are equal to the marker phase types from gecko-profile.js
+    module Phase # :nodoc:
+      INSTANT = 0
+      INTERVAL = 1
+      INTERVAL_START = 2
+      INTERVAL_END = 3
+    end
+
+    MARKER_STRINGS = []
+
+    MARKER_STRINGS[Type::GVL_THREAD_STARTED] = "thread started"
+    MARKER_STRINGS[Type::GVL_THREAD_READY] = "thread ready"
+    MARKER_STRINGS[Type::GVL_THREAD_RESUMED] = "thread resumed"
+    MARKER_STRINGS[Type::GVL_THREAD_SUSPENDED] = "thread suspended"
+    MARKER_STRINGS[Type::GVL_THREAD_EXITED] = "thread exited"
+
+    MARKER_STRINGS[Type::GC_START] = "GC start"
+    MARKER_STRINGS[Type::GC_END_MARK] = "GC end marking"
+    MARKER_STRINGS[Type::GC_END_SWEEP] = "GC end sweeping"
+    MARKER_STRINGS[Type::GC_ENTER] = "GC enter"
+    MARKER_STRINGS[Type::GC_EXIT] = "GC exit"
+
+    MARKER_STRINGS.freeze
+
+    ##
+    # Return an array of marker names.  The index of the string maps to the
+    # value of the corresponding constant
+    def self.name_table
+      MARKER_STRINGS
+    end
+  end
+end

data/lib/vernier/output/firefox.rb

@@ -0,0 +1,365 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Vernier
+  module Output
+    # https://profiler.firefox.com/
+    # https://github.com/firefox-devtools/profiler/blob/main/src/types/profile.js
+    class Firefox
+      class Categorizer
+        attr_reader :categories, :gc_category, :jit_category
+        def initialize
+          @categories = []
+
+          add_category(name: "Default", color: "grey")
+          add_category(name: "Idle", color: "transparent")
+          @gc_category = add_category(name: "GC", color: "red")
+          add_category(
+            name: "stdlib",
+            color: "red",
+            matcher: starts_with(RbConfig::CONFIG["rubylibdir"])
+          )
+          add_category(name: "cfunc", color: "yellow", matcher: "<cfunc>")
+
+          rails_components = %w[ activesupport activemodel activerecord
+          actionview actionpack activejob actionmailer actioncable
+          activestorage actionmailbox actiontext railties ]
+          add_category(
+            name: "Rails",
+            color: "green",
+            matcher: gem_path(*rails_components)
+          )
+          add_category(
+            name: "gem",
+            color: "red",
+            matcher: starts_with(*Gem.path)
+          )
+          add_category(
+            name: "Application",
+            color: "purple"
+          )
+          @jit_category = add_category(
+            name: "JIT",
+            color: "blue"
+          )
+        end
+
+        def add_category(**kw)
+          category = Category.new(@categories.length, **kw)
+          @categories << category
+          category
+        end
+
+        def starts_with(*paths)
+          %r{\A#{Regexp.union(paths)}}
+        end
+
+        def gem_path(*names)
+          %r{\A#{Regexp.union(Gem.path)}/gems/#{Regexp.union(names)}}
+        end
+
+        def categorize(path)
+          @categories.detect { |category| category.matches?(path) } || @categories.first
+        end
+
+        class Category
+          attr_reader :idx, :name, :color, :matcher
+          def initialize(idx, name:, color:, matcher: nil)
+            @idx = idx
+            @name = name
+            @color = color
+            @matcher = matcher
+          end
+
+          def matches?(path)
+            @matcher && @matcher === path
+          end
+        end
+      end
+
+      def initialize(profile)
+        @profile = profile
+        @categorizer = Categorizer.new
+      end
+
+      def output
+        ::JSON.generate(data)
+      end
+
+      private
+
+      attr_reader :profile
+
+      def data
+        threads = Hash.new {|h,k| h[k] = {
+          timestamps: [],
+          weights: [],
+          samples: [],
+          categories: [],
+          markers: [],
+        }}
+
+        profile.samples.size.times do |i|
+          tid = profile.sample_threads[i]
+          thread = threads[tid]
+
+          thread[:timestamps] << profile.timestamps[i]
+          thread[:weights] << profile.weights[i]
+          thread[:samples] << profile.samples[i]
+          thread[:categories] << profile.sample_categories[i]
+        end
+
+        profile.markers.each do |marker|
+          threads[marker[0]][:markers] << marker
+        end
+
+        thread_data = profile.threads.map do |tid, thread_info|
+          data = threads[tid]
+          Thread.new(
+            profile,
+            @categorizer,
+            **thread_info,
+            **data
+          ).data
+        end
+
+        {
+          meta: {
+            interval: 1, # FIXME: memory vs wall
+            startTime: profile.meta[:started_at] / 1_000_000.0,
+            endTime: (profile.timestamps&.max || 0) / 1_000_000.0,
+            processType: 0,
+            product: "Ruby/Vernier",
+            stackwalk: 1,
+            version: 28,
+            preprocessedProfileVersion: 47,
+            symbolicated: true,
+            markerSchema: [],
+            sampleUnits: {
+              time: "ms",
+              eventDelay: "ms",
+              threadCPUDelta: "µs"
+            }, # FIXME: memory vs wall
+            categories: @categorizer.categories.map do |category|
+              {
+                name: category.name,
+                color: category.color,
+                subcategories: []
+              }
+            end
+          },
+          libs: [],
+          threads: thread_data
+        }
+      end
+
+      class Thread
+        attr_reader :profile
+
+        def initialize(profile, categorizer, name:, tid:, samples:, weights:, timestamps:, categories:, markers:, started_at:, stopped_at: nil)
+          @profile = profile
+          @categorizer = categorizer
+          @tid = tid
+          @name = name
+
+          @samples, @weights, @timestamps = samples, weights, timestamps
+          @sample_categories = categories
+          @markers = markers
+
+          @started_at, @stopped_at = started_at, stopped_at
+
+          names = profile.func_table.fetch(:name)
+          filenames = profile.func_table.fetch(:filename)
+
+          stacks_size = profile.stack_table.fetch(:frame).size
+          @categorized_stacks = Hash.new do |h, k|
+            h[k] = h.size + stacks_size
+          end
+
+          @strings = Hash.new { |h, k| h[k] = h.size }
+          @func_names = names.map do |name|
+            @strings[name]
+          end
+          @filenames = filenames.map do |filename|
+            @strings[filename]
+          end
+          @frame_categories = filenames.map do |filename|
+            @categorizer.categorize(filename)
+          end
+        end
+
+        def data
+          {
+            name: @name,
+            isMainThread: @tid == ::Thread.main.native_thread_id,
+            processStartupTime: 0, # FIXME
+            processShutdownTime: nil, # FIXME
+            registerTime: (@started_at - 0) / 1_000_000.0,
+            unregisterTime: ((@stopped_at - 0) / 1_000_000.0 if @stopped_at),
+            pausedRanges: [],
+            pid: profile.pid || Process.pid,
+            tid: @tid,
+            frameTable: frame_table,
+            funcTable: func_table,
+            nativeSymbols: {},
+            samples: samples_table,
+            stackTable: stack_table,
+            resourceTable: {
+              length: 0,
+              lib: [],
+              name: [],
+              host: [],
+              type: []
+            },
+            markers: markers_table,
+            stringArray: string_table
+          }
+        end
+
+        def markers_table
+          size = @markers.size
+
+          string_indexes = []
+          start_times = []
+          end_times = []
+          phases = []
+          categories = []
+
+          @markers.each_with_index do |(_, name, start, finish, phase), i|
+            string_indexes << @strings[name]
+            start_times << (start / 1_000_000.0)
+
+            # Please don't hate me. Divide by 1,000,000 only if finish is not nil
+            end_times << (finish&./(1_000_000.0))
+            phases << phase
+            categories << (name =~ /GC/ ? gc_category.idx : 0)
+          end
+
+          {
+            data: [nil] * start_times.size,
+            name: string_indexes,
+            startTime: start_times,
+            endTime: end_times,
+            phase: phases,
+            category: categories,
+            length: start_times.size
+          }
+        end
+
+        def samples_table
+          samples = @samples
+          weights = @weights
+          categories = @sample_categories
+          size = samples.size
+          if categories.empty?
+            categories = [0] * size
+          end
+
+          if @timestamps
+            times = @timestamps.map { _1 / 1_000_000.0 }
+          else
+            # FIXME: record timestamps for memory samples
+            times = (0...size).to_a
+          end
+
+          raise unless samples.size == size
+          raise unless weights.size == size
+          raise unless times.size == size
+
+          samples = samples.zip(categories).map do |sample, category|
+            if category == 0
+              sample
+            else
+              @categorized_stacks[[sample, category]]
+            end
+          end
+
+          {
+            stack: samples,
+            time: times,
+            weight: weights,
+            weightType: "samples",
+            #weightType: "bytes",
+            length: samples.length
+          }
+        end
+
+        def stack_table
+          frames = profile.stack_table.fetch(:frame).dup
+          prefixes = profile.stack_table.fetch(:parent).dup
+          categories  = frames.map{|idx| @frame_categories[idx].idx }
+
+          @categorized_stacks.keys.each do |(stack, category)|
+            frames << frames[stack]
+            prefixes << prefixes[stack]
+            categories << category
+          end
+
+          size = frames.length
+          raise unless frames.size == size
+          raise unless prefixes.size == size
+          {
+            frame: frames,
+            category: categories,
+            subcategory: [0] * size,
+            prefix: prefixes,
+            length: prefixes.length
+          }
+        end
+
+        def frame_table
+          funcs = profile.frame_table.fetch(:func)
+          lines = profile.frame_table.fetch(:line)
+          size = funcs.length
+          none = [nil] * size
+          categories = @frame_categories.map(&:idx)
+
+          raise unless lines.size == funcs.size
+
+          {
+            address: [-1] * size,
+            inlineDepth: [0] * size,
+            category: categories,
+            subcategory: nil,
+            func: funcs,
+            nativeSymbol: none,
+            innerWindowID: none,
+            implementation: none,
+            line: lines,
+            column: none,
+            length: size
+          }
+        end
+
+        def func_table
+          size = @func_names.size
+
+          cfunc_idx = @strings["<cfunc>"]
+          is_js = @filenames.map { |fn| fn != cfunc_idx }
+          {
+            name: @func_names,
+            isJS: is_js,
+            relevantForJS: is_js,
+            resource: [-1] * size, # set to unidentified for now
+            fileName: @filenames,
+            lineNumber: profile.func_table.fetch(:first_line),
+            columnNumber: [0] * size,
+            #columnNumber: functions.map { _1.column },
+            length: size
+          }
+        end
+
+        def string_table
+          @strings.keys
+        end
+
+        private
+
+        def gc_category
+          @categorizer.gc_category
+        end
+      end
+    end
+  end
+end

data/lib/vernier/output/top.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Vernier
+  module Output
+    class Top
+      def initialize(profile)
+        @profile = profile
+      end
+
+      def output
+        stack_weights = Hash.new(0)
+        @profile.samples.zip(@profile.weights) do |stack_idx, weight|
+          stack_weights[stack_idx] += weight
+        end
+
+        top_by_self = Hash.new(0)
+        stack_weights.each do |stack_idx, weight|
+          stack = @profile.stack(stack_idx)
+          top_by_self[stack.leaf_frame.name] += weight
+        end
+
+        s = +""
+        top_by_self.sort_by(&:last).reverse.each do |frame, samples|
+          s << "#{samples}\t#{frame}\n"
+        end
+        s
+      end
+    end
+  end
+end

data/lib/vernier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vernier
-  VERSION = "0.1.999"
+  VERSION = "0.2.0"
 end

data/lib/vernier.rb

@@ -1,19 +1,181 @@
 # frozen_string_literal: true
 
 require_relative "vernier/version"
+require_relative "vernier/collector"
 require_relative "vernier/vernier"
+require_relative "vernier/output/firefox"
+require_relative "vernier/output/top"
 
 module Vernier
   class Error < StandardError; end
 
+  class Result
+    attr_reader :weights, :samples, :stack_table, :frame_table, :func_table
+    attr_reader :timestamps, :sample_threads, :sample_categories
+    attr_reader :markers
+
+    attr_accessor :pid, :start_time, :end_time
+    attr_accessor :threads
+    attr_accessor :meta
+
+    def started_at
+      meta[:started_at]
+    end
+
+    def to_gecko
+      Output::Firefox.new(self).output
+    end
+
+    def write(out:)
+      File.write(out, to_gecko)
+    end
+
+    def each_sample
+      return enum_for(__method__) unless block_given?
+      @samples.size.times do |sample_idx|
+        weight = @weights[sample_idx]
+        stack_idx = @samples[sample_idx]
+        yield stack(stack_idx), weight
+      end
+    end
+
+    class BaseType
+      attr_reader :result, :idx
+      def initialize(result, idx)
+        @result = result
+        @idx = idx
+      end
+
+      def to_s
+        idx.to_s
+      end
+
+      def inspect
+        "#<#{self.class}\n#{to_s}>"
+      end
+    end
+
+    class Func < BaseType
+      def label
+        result.func_table[:name][idx]
+      end
+      alias name label
+
+      def filename
+        result.func_table[:filename][idx]
+      end
+
+      def to_s
+        "#{name} at #{filename}"
+      end
+    end
+
+    class Frame < BaseType
+      def label; func.label; end
+      def filename; func.filename; end
+      alias name label
+
+      def func
+        func_idx = result.frame_table[:func][idx]
+        Func.new(result, func_idx)
+      end
+
+      def line
+        result.frame_table[:line][idx]
+      end
+
+      def to_s
+        "#{func}:#{line}"
+      end
+    end
+
+    class Stack < BaseType
+      def each_frame
+        return enum_for(__method__) unless block_given?
+
+        stack_idx = idx
+        while stack_idx
+          frame_idx = result.stack_table[:frame][stack_idx]
+          yield Frame.new(result, frame_idx)
+          stack_idx = result.stack_table[:parent][stack_idx]
+        end
+      end
+
+      def leaf_frame_idx
+        result.stack_table[:frame][idx]
+      end
+
+      def leaf_frame
+        Frame.new(result, leaf_frame_idx)
+      end
+
+      def frames
+        each_frame.to_a
+      end
+
+      def to_s
+        arr = []
+        each_frame do |frame|
+          arr << frame.to_s
+        end
+        arr.join("\n")
+      end
+    end
+
+    def stack(idx)
+      Stack.new(self, idx)
+    end
+
+    def total_bytes
+      @weights.sum
+    end
+  end
+
+  def self.trace(mode: :wall, out: nil, interval: nil)
+    collector = Vernier::Collector.new(mode, { interval: })
+    collector.start
+
+    result = nil
+    begin
+      yield
+    ensure
+      result = collector.stop
+    end
+
+    if out
+      File.write(out, Output::Firefox.new(result).output)
+    end
+    result
+  end
+
   def self.trace_retained(out: nil, gc: true)
     3.times { GC.start } if gc
-    Vernier.trace_retained_start
-    yield
-    3.times { GC.start } if gc
-    result = Vernier.trace_retained_stop
 
-    File.write(out, result) if out
+    start_time = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+
+    collector = Vernier::Collector.new(:retained)
+    collector.start
+
+    result = nil
+    begin
+      yield
+    ensure
+      result = collector.stop
+      end_time = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+      result.pid = Process.pid
+      result.start_time = start_time
+      result.end_time = end_time
+    end
+
+    if out
+      result.write(out:)
+    end
     result
   end
+
+  class Collector
+    def self.new(mode, options = {})
+      _new(mode, options)
+    end
+  end
 end

data/vernier.gemspec

@@ -12,19 +12,12 @@ Gem::Specification.new do |spec|
   spec.description = spec.summary
   spec.homepage = "https://github.com/jhawthorn/vernier"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = ">= 3.2.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
   spec.metadata["changelog_uri"] = spec.homepage
 
-  spec.post_install_message = <<~WARNING
-    #{'!'*80}
-    WARNING! You've installed a very old version of Vernier, likely due to Ruby >= 3.2 version requirement.
-    Please use gem "vernier", "~> 1.0" in your gemspec to install a modern version.
-    #{'!'*80}
-  WARNING
-
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do