heap-profiler 0.1.0

data/lib/heap_profiler/monochrome.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  module Monochrome
+    class << self
+      def path(text)
+        text
+      end
+
+      def string(text)
+        text
+      end
+
+      def line(text)
+        text
+      end
+    end
+  end
+end

data/lib/heap_profiler/native.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  module Native
+    DEFAULT_BATCH_SIZE = 10_000_000 # 10MB
+    class << self
+      def build_index(path, batch_size: DEFAULT_BATCH_SIZE)
+        _build_index(path, batch_size)
+      end
+
+      def load_many(path, batch_size: DEFAULT_BATCH_SIZE, &block)
+        _load_many(path, batch_size, &block)
+      end
+
+      def filter_heap(source_path, destination_path, since:)
+        _filter_heap(source_path, destination_path, since)
+      end
+
+      def ruby_build_index(path)
+        require 'json'
+        classes_index = {}
+        strings_index = {}
+
+        File.open(path).each_line do |line|
+          object = JSON.parse(line, symbolize_names: true)
+          case object[:type]
+          when 'MODULE', 'CLASS'
+            if (name = object[:name])
+              classes_index[ruby_parse_address(object[:address])] = name
+            end
+          when 'STRING'
+            next if object[:shared]
+            if (value = object[:value])
+              strings_index[ruby_parse_address(object[:address])] = value
+            end
+          end
+        end
+
+        [classes_index, strings_index]
+      end
+
+      def ruby_parse_address(address)
+        address.to_i(16)
+      end
+    end
+  end
+  require "heap_profiler/heap_profiler"
+end

data/lib/heap_profiler/polychrome.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  module Polychrome
+    class << self
+      def path(text)
+        blue(text)
+      end
+
+      def string(text)
+        green(text)
+      end
+
+      def line(text)
+        cyan(text)
+      end
+
+      private
+
+      def black(str)
+        "\033[30m#{str}\033[0m"
+      end
+
+      def red(str)
+        "\033[31m#{str}\033[0m"
+      end
+
+      def green(str)
+        "\033[32m#{str}\033[0m"
+      end
+
+      def brown(str)
+        "\033[33m#{str}\033[0m"
+      end
+
+      def blue(str)
+        "\033[34m#{str}\033[0m"
+      end
+
+      def magenta(str)
+        "\033[35m#{str}\033[0m"
+      end
+
+      def cyan(str)
+        "\033[36m#{str}\033[0m"
+      end
+
+      def gray(str)
+        "\033[37m#{str}\033[0m"
+      end
+
+      def bg_black(str)
+        "\033[40m#{str}\033[0m"
+      end
+
+      def bg_red(str)
+        "\033[41m#{str}\033[0m"
+      end
+
+      def bg_green(str)
+        "\033[42m#{str}\033[0m"
+      end
+
+      def bg_brown(str)
+        "\033[43m#{str}\033[0m"
+      end
+
+      def bg_blue(str)
+        "\033[44m#{str}\033[0m"
+      end
+
+      def bg_magenta(str)
+        "\033[45m#{str}\033[0m"
+      end
+
+      def bg_cyan(str)
+        "\033[46m#{str}\033[0m"
+      end
+
+      def bg_gray(str)
+        "\033[47m#{str}\033[0m"
+      end
+
+      def bold(str)
+        "\033[1m#{str}\033[22m"
+      end
+
+      def reverse_color(str)
+        "\033[7m#{str}\033[27m"
+      end
+    end
+  end
+end

data/lib/heap_profiler/reporter.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class << self
+    # This works around a Ruby bug present until at least 2.7.1
+    # ObjectSpace.dump include module and class names in the dump
+    # and for anonymous modules and classes this mean naming them.
+    #
+    # So we name them at the start of the profile to avoid that.
+    #
+    # See: https://github.com/ruby/ruby/pull/3349
+    #
+    # TODO: Could we actually do the dump ourselves? objspace is a extension already.
+    if RUBY_VERSION < '2.8'
+      def name_anonymous_modules!
+        ObjectSpace.each_object(Module) do |mod|
+          next if mod.singleton_class?
+          next if real_mod_name(mod)
+          # We have to assign it at the top level to avoid allocating a string for the name
+          ::Object.const_set(:AnonymousClassOrModule, mod)
+          ::Object.send(:remove_const, :AnonymousClassOrModule)
+        end
+      end
+
+      ::Module.alias_method(:__real_mod_name, :name)
+      def real_mod_name(mod)
+        mod.__real_mod_name
+      end
+    else
+      def name_anonymous_modules!
+      end
+    end
+  end
+
+  class Reporter
+    def initialize(dir_path)
+      @dir_path = dir_path
+      @enable_tracing = !allocation_tracing_enabled?
+      @generation = nil
+    end
+
+    def start
+      FileUtils.mkdir_p(@dir_path)
+      ObjectSpace.trace_object_allocations_start if @enable_tracing
+
+      @allocated_heap = open_heap("allocated")
+      @retained_heap = open_heap("retained")
+
+      HeapProfiler.name_anonymous_modules!
+
+      4.times { GC.start }
+      GC.disable
+      @generation = GC.count
+    end
+
+    def stop
+      HeapProfiler.name_anonymous_modules!
+      ObjectSpace.trace_object_allocations_stop if @enable_tracing
+      dump_heap(@allocated_heap)
+      GC.enable
+      4.times { GC.start }
+      dump_heap(@retained_heap)
+      @allocated_heap.close
+      @retained_heap.close
+      write_info("generation", @generation.to_s)
+    end
+
+    def run
+      start
+      begin
+        yield
+      rescue Exception
+        ObjectSpace.trace_object_allocations_stop if @enable_tracing
+        GC.enable
+        raise
+      else
+        stop
+      end
+    end
+
+    private
+
+    def write_info(key, value)
+      File.write(File.join(@dir_path, "#{key}.info"), value)
+    end
+
+    # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
+    # because of this even en empty block of code will report a handful of allocations.
+    # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
+    # property.
+    class_eval <<~RUBY, '__hprof', __LINE__
+      # frozen_string_literal: true
+      def dump_heap(file)
+        ObjectSpace.dump_all(output: file)
+        file.close
+      end
+    RUBY
+
+    def open_heap(name)
+      File.open(File.join(@dir_path, "#{name}.heap"), 'w+')
+    end
+
+    def allocation_tracing_enabled?
+      ObjectSpace.allocation_sourceline(Object.new)
+    end
+  end
+end

data/lib/heap_profiler/results.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class AbstractResults
+    UNIT_PREFIXES = {
+      0 => 'B',
+      3 => 'kB',
+      6 => 'MB',
+      9 => 'GB',
+      12 => 'TB',
+      15 => 'PB',
+      18 => 'EB',
+      21 => 'ZB',
+      24 => 'YB',
+    }.freeze
+
+    METRICS = ["memory", "objects", "strings"].freeze
+    GROUPINGS = ["gem", "file", "location", "class"].freeze
+
+    attr_reader :types, :dimensions
+
+    def initialize(*, **)
+      raise NotImplementedError
+    end
+
+    def print_title(io, title)
+      io.puts
+      io.puts title
+      io.puts @colorize.line("-----------------------------------")
+    end
+
+    def print_output(io, topic, detail)
+      io.puts "#{@colorize.path(topic.to_s.rjust(10))}  #{detail}"
+    end
+
+    def print_output2(io, topic1, topic2, detail)
+      io.puts "#{@colorize.path(topic1.to_s.rjust(10))}  #{@colorize.path(topic2.to_s.rjust(6))}  #{detail}"
+    end
+
+    def normalize_path(path)
+      @normalize_path ||= {}
+      @normalize_path[path] ||= begin
+        if %r!(/gems/.*)*/gems/(?<gemname>[^/]+)(?<rest>.*)! =~ path
+          "#{gemname}#{rest}"
+        elsif %r!ruby/2\.[^/]+/(?<stdlib>[^/.]+)(?<rest>.*)! =~ path
+          "ruby/lib/#{stdlib}#{rest}"
+        elsif %r!(?<app>[^/]+/(bin|app|lib))(?<rest>.*)! =~ path
+          "#{app}#{rest}"
+        else
+          path
+        end
+      end
+    end
+
+    def scale_bytes(bytes)
+      return "0 B" if bytes.zero?
+
+      scale = Math.log10(bytes).div(3) * 3
+      scale = 24 if scale > 24
+      format("%.2f #{UNIT_PREFIXES[scale]}", (bytes / 10.0**scale))
+    end
+  end
+
+  class HeapResults < AbstractResults
+    def initialize(heap_path, metrics = METRICS, groupings = GROUPINGS)
+      @path = heap_path
+      @metrics = metrics
+      @groupings = groupings
+    end
+
+    def pretty_print(io = $stdout, **options)
+      heap = Dump.new(@path)
+      index = Index.new(heap)
+
+      color_output = options.fetch(:color_output) { io.respond_to?(:isatty) && io.isatty }
+      @colorize = color_output ? Polychrome : Monochrome
+
+      analyzer = Analyzer.new(heap, index)
+      dimensions = analyzer.run(@metrics, @groupings)
+
+      io.puts "Total: #{scale_bytes(dimensions['total_memory'].stats)} " \
+              "(#{dimensions['total_objects'].stats} objects)"
+
+      @metrics.each do |metric|
+        next if metric == "strings"
+        @groupings.each do |grouping|
+          dump_data(io, dimensions, metric, grouping, options)
+        end
+      end
+
+      if @metrics.include?("strings")
+        dump_strings(io, dimensions, options)
+      end
+    end
+
+    def dump_data(io, dimensions, metric, grouping, options)
+      print_title io, "#{metric} by #{grouping}"
+      data = dimensions["#{metric}_by_#{grouping}"].top_n(options.fetch(:top, 50))
+
+      scale_data = metric == "memory" && options[:scale_bytes]
+      normalize_paths = options[:normalize_paths]
+
+      if data && !data.empty?
+        data.each { |pair| pair[0] = normalize_path(pair[0]) } if normalize_paths
+        data.each { |pair| pair[1] = scale_bytes(pair[1]) } if scale_data
+        data.each { |k, v| print_output(io, v, k) }
+      else
+        io.puts "NO DATA"
+      end
+    end
+
+    def dump_strings(io, dimensions, options)
+      normalize_paths = options[:normalize_paths]
+      scale_data = options[:scale_bytes]
+      top = options.fetch(:top, 50)
+
+      print_title(io, "String Report")
+
+      dimensions["strings"].top_n(top).each do |string|
+        memsize = scale_data ? scale_bytes(string.memsize) : string.memsize
+        print_output2 io, memsize, string.count, @colorize.string(string.value.inspect)
+        string.top_n(top).each do |string_location|
+          location = string_location.location
+          location = normalize_path(location) if normalize_paths
+          print_output2 io, '', string_location.count, location
+        end
+        io.puts
+      end
+    end
+  end
+
+  class DiffResults < AbstractResults
+    TYPES = ["allocated", "retained"].freeze
+
+    def initialize(directory, types = TYPES, metrics = METRICS, groupings = GROUPINGS)
+      @directory = directory
+      @types = types
+      @metrics = metrics
+      @groupings = groupings
+    end
+
+    def pretty_print(io = $stdout, **options)
+      diff = Diff.new(@directory)
+      heaps = @types.to_h { |t| [t, diff.public_send("#{t}_diff")] }
+      index = Index.new(diff.allocated)
+
+      color_output = options.fetch(:color_output) { io.respond_to?(:isatty) && io.isatty }
+      @colorize = color_output ? Polychrome : Monochrome
+
+      dimensions = {}
+      heaps.each do |type, heap|
+        analyzer = Analyzer.new(heap, index)
+        dimensions[type] = analyzer.run(@metrics, @groupings)
+      end
+
+      dimensions.each do |type, metrics|
+        io.puts "Total #{type}: #{scale_bytes(metrics['total_memory'].stats)} " \
+                "(#{metrics['total_objects'].stats} objects)"
+      end
+
+      @types.each do |type|
+        @metrics.each do |metric|
+          next if metric == "strings"
+          @groupings.each do |grouping|
+            dump_data(io, dimensions, type, metric, grouping, options)
+          end
+        end
+      end
+
+      if @metrics.include?("strings")
+        @types.each do |type|
+          dump_strings(io, dimensions[type], type, options)
+        end
+      end
+    end
+
+    def dump_data(io, dimensions, type, metric, grouping, options)
+      print_title io, "#{type} #{metric} by #{grouping}"
+      data = dimensions[type]["#{metric}_by_#{grouping}"].top_n(options.fetch(:top, 50))
+
+      scale_data = metric == "memory" && options[:scale_bytes]
+      normalize_paths = options[:normalize_paths]
+
+      if data && !data.empty?
+        data.each { |pair| pair[0] = normalize_path(pair[0]) } if normalize_paths
+        data.each { |pair| pair[1] = scale_bytes(pair[1]) } if scale_data
+        data.each { |k, v| print_output(io, v, k) }
+      else
+        io.puts "NO DATA"
+      end
+    end
+
+    def dump_strings(io, dimensions, type, options)
+      normalize_paths = options[:normalize_paths]
+      scale_data = options[:scale_bytes]
+      top = options.fetch(:top, 50)
+
+      print_title(io, "#{type.capitalize} String Report")
+
+      dimensions["strings"].top_n(top).each do |string|
+        memsize = scale_data ? scale_bytes(string.memsize) : string.memsize
+        print_output2 io, memsize, string.count, @colorize.string(string.value.inspect)
+        string.top_n(top).each do |string_location|
+          location = string_location.location
+          location = normalize_path(location) if normalize_paths
+          print_output2 io, '', string_location.count, location
+        end
+        io.puts
+      end
+    end
+  end
+end