heap-profiler 0.1.0

data/heap-profiler.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require_relative 'lib/heap_profiler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "heap-profiler"
+  spec.version       = Heap::Profiler::VERSION
+  spec.authors       = ["Jean Boussier"]
+  spec.email         = ["jean.boussier@gmail.com"]
+
+  spec.summary       = 'Ruby heap profiling tool'
+  spec.description   = 'Make several heap dumps and summarize allocated, retained memory'
+  spec.homepage      = "https://github.com/Shopify/heap-profiler"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+    %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.extensions << "ext/heap_profiler/extconf.rb"
+end

data/lib/heap-profiler.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# This file is the gem entrypoint loaded by bundler.
+# We only load what's needed to take heap dumps,
+# not the code required to analyse them.
+require "heap_profiler/runtime"

data/lib/heap_profiler/analyzer.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class Analyzer
+    class Dimension
+      METRICS = {
+        "objects" => -> (_object) { 1 },
+        "memory" => -> (object) { object[:memsize].to_i },
+      }.freeze
+
+      attr_reader :stats
+      def initialize(metric)
+        @stats = 0
+        @metric = METRICS.fetch(metric)
+      end
+
+      def process(_index, object)
+        @stats += @metric.call(object)
+      end
+
+      def sort!
+      end
+    end
+
+    class GroupedDimension < Dimension
+      GROUPINGS = {
+        "file" => -> (_index, object) { object[:file] },
+        "location" => -> (_index, object) do
+          if (file = object[:file]) && (line = object[:line])
+            "#{file}:#{line}"
+          end
+        end,
+        "gem" => -> (index, object) { index.guess_gem(object[:file]) },
+        "class" => -> (index, object) { index.guess_class(object) },
+      }.freeze
+
+      attr_reader :stats
+      def initialize(metric, grouping)
+        super(metric)
+        @grouping = GROUPINGS.fetch(grouping)
+        @stats = Hash.new { |h, k| h[k] = 0 }
+      end
+
+      def process(index, object)
+        if (group = @grouping.call(index, object))
+          @stats[group] += @metric.call(object)
+        end
+      end
+
+      def top_n(max)
+        stats.sort do |a, b|
+          cmp = b[1] <=> a[1]
+          cmp == 0 ? b[0] <=> a[0] : cmp
+        end.take(max)
+      end
+    end
+
+    class StringDimension
+      class StringLocation
+        attr_reader :location, :count, :memsize
+
+        def initialize(location)
+          @location = location
+          @count = 0
+          @memsize = 0
+        end
+
+        def process(object)
+          @count += 1
+          @memsize += object[:memsize]
+        end
+      end
+
+      class StringGroup
+        attr_reader :value, :count, :memsize, :locations
+        def initialize(value) # TODO: should we consider encoding?
+          @value = value
+          @locations_counts = Hash.new { |h, k| h[k] = StringLocation.new(k) }
+          @count = 0
+          @memsize = 0
+        end
+
+        def process(object)
+          @count += 1
+          @memsize += object[:memsize]
+          if (file = object[:file]) && (line = object[:line])
+            @locations_counts["#{file}:#{line}"].process(object)
+          end
+        end
+
+        def top_n(_max)
+          values = @locations_counts.values
+          values.sort! do |a, b|
+            cmp = b.count <=> a.count
+            cmp == 0 ? b.location <=> a.location : cmp
+          end
+        end
+      end
+
+      attr_reader :stats
+      def initialize
+        @stats = Hash.new { |h, k| h[k] = StringGroup.new(k) }
+      end
+
+      def process(_index, object)
+        return unless object[:type] == "STRING"
+        value = object[:value]
+        return unless value # broken strings etc
+        @stats[value].process(object)
+      end
+
+      def top_n(max)
+        values = @stats.values
+        values.sort! do |a, b|
+          cmp = b.count <=> a.count
+          cmp == 0 ? b.value <=> a.value : cmp
+        end
+        values.take(max)
+      end
+    end
+
+    def initialize(heap, index)
+      @heap = heap
+      @index = index
+    end
+
+    def run(metrics, groupings)
+      dimensions = {}
+      metrics.each do |metric|
+        if metric == "strings"
+          dimensions["strings"] = StringDimension.new
+        else
+          dimensions["total_#{metric}"] = Dimension.new(metric)
+          groupings.each do |grouping|
+            dimensions["#{metric}_by_#{grouping}"] = GroupedDimension.new(metric, grouping)
+          end
+        end
+      end
+
+      processors = dimensions.values
+      @heap.each_object do |object|
+        processors.each { |p| p.process(@index, object) }
+      end
+      dimensions
+    end
+  end
+end

data/lib/heap_profiler/cli.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class CLI
+    def initialize(argv)
+      @argv = argv
+    end
+
+    def run
+      if @argv.size == 1
+        print_report(@argv.first)
+        0
+      else
+        print_usage
+        1
+      end
+    end
+
+    def print_report(path)
+      results = if File.directory?(path)
+        DiffResults.new(path)
+      else
+        HeapResults.new(path)
+      end
+      results.pretty_print(scale_bytes: true, normalize_paths: true)
+    end
+
+    def print_usage
+      puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
+    end
+  end
+end

data/lib/heap_profiler/diff.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class Diff
+    attr_reader :allocated
+
+    def initialize(report_directory)
+      @report_directory = report_directory
+      @allocated = open_dump('allocated')
+      @generation = Integer(File.read(File.join(report_directory, 'generation.info')))
+    end
+
+    def allocated_diff
+      @allocated_diff ||= build_diff('allocated-diff', @allocated)
+    end
+
+    def retained_diff
+      @retained_diff ||= build_diff('retained-diff', open_dump('retained'))
+    end
+
+    private
+
+    def build_diff(name, base)
+      diff = open_dump(name)
+      unless diff.exist?
+        base.filter(File.join(@report_directory, "#{name}.heap"), since: @generation)
+      end
+      diff
+    end
+
+    def open_dump(name)
+      Dump.open(@report_directory, name)
+    end
+  end
+end

data/lib/heap_profiler/dump.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class Dump
+    class Stats
+      attr_accessor :count, :memsize
+
+      def process(object)
+        @count += 1
+        @memsize += object.fetch(:memsize, 0)
+      end
+
+      def initialize
+        @count = 0
+        @memsize = 0
+      end
+    end
+
+    class GlobalStats < Stats
+      class << self
+        def from(dump)
+          stats = new
+          dump.each_object do |object|
+            stats.process(object)
+          end
+          stats
+        end
+      end
+
+      def process(object)
+        super
+        per_type[object[:type]].process(object)
+      end
+
+      def per_type
+        @per_type = Hash.new { |h, k| h[k] = Stats.new }
+      end
+    end
+
+    class << self
+      def open(dir, name)
+        Dump.new(File.join(dir, "#{name}.heap"))
+      end
+    end
+
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    # ObjectSpace.dump_all itself allocate objects.
+    #
+    # Before 2.7 it will allocate one String per class to get its name.
+    # After 2.7, it only allocate a couple hashes, a file etc.
+    #
+    # Either way we need to exclude them from the reports
+    def diff(other, file)
+      each_line_with_address do |line, address|
+        file << line unless other.index.include?(address)
+      end
+    end
+
+    def filter(as, since:)
+      Native.filter_heap(path, as, since: since)
+    end
+
+    def each_object(&block)
+      Native.load_many(path, batch_size: 10_000_000, &block)
+    end
+
+    def stats
+      @stats ||= GlobalStats.from(self)
+    end
+
+    def size
+      @size ||= File.open(path).each_line.count
+    end
+
+    def index
+      @index ||= Native.addresses_set(path)
+    end
+
+    def each_line_with_address
+      File.open(path).each_line do |line|
+        # This is a cheap, hacky extraction of addresses.
+        # So far it seems to work on 2.7.1 but that might not hold true on all versions.
+        # Also the root objects don't have an address, but that's fine
+        yield line, line.byteslice(14, 12).to_i(16)
+      end
+    end
+
+    def exist?
+      File.exist?(@path)
+    end
+
+    def presence
+      exist? ? self : nil
+    end
+  end
+end

data/lib/heap_profiler/full.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "heap_profiler/runtime"
+require "heap_profiler/native"
+require "heap_profiler/dump"
+require "heap_profiler/index"
+require "heap_profiler/diff"
+require "heap_profiler/analyzer"
+require "heap_profiler/polychrome"
+require "heap_profiler/monochrome"
+require "heap_profiler/results"
+require "heap_profiler/cli"

data/lib/heap_profiler/index.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class Index
+    def initialize(heap)
+      @heap = heap
+      @classes = {}
+      @strings = {}
+      @gems = {}
+      build!
+    end
+
+    def build!
+      @classes, @strings = Native.build_index(@heap.path)
+      self
+    end
+
+    BUILTIN_CLASSES = {
+      "FILE" => "File",
+      "ICLASS" => "ICLASS",
+      "COMPLEX" => "Complex",
+      "RATIONAL" => "Rational",
+      "BIGNUM" => "Bignum",
+      "FLOAT" => "Float",
+      "ARRAY" => "Array",
+      "STRING" => "String",
+      "HASH" => "Hash",
+      "SYMBOL" => "Symbol",
+      "MODULE" => "Module",
+      "CLASS" => "Class",
+      "REGEXP" => "Regexp",
+      "MATCH" => "MatchData",
+      "ROOT" => "<VM Root>",
+    }.freeze
+
+    IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }
+    DATA_TYPES = Hash.new { |h, k| h[k] = "<#{(k || 'unknown')}> (DATA)" }
+
+    def guess_class(object)
+      type = object[:type]
+      if (class_name = BUILTIN_CLASSES[type])
+        return class_name
+      end
+
+      return IMEMO_TYPES[object[:imemo_type]] if type == 'IMEMO'
+      return DATA_TYPES[object[:struct]] if type == 'DATA'
+
+      if type == "OBJECT" || type == "STRUCT"
+        class_address = object[:class]
+        return unless class_address
+
+        return @classes.fetch(class_address) do
+          $stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
+          nil
+        end
+      end
+
+      raise "[BUG] Couldn't infer type of #{object.inspect}"
+    end
+
+    def string_value(object)
+      value = object[:value]
+      return value if value
+
+      if object[:shared]
+        @strings[cast_address(object[:references].first)]
+      end
+    end
+
+    def cast_address(address)
+      address.to_s.to_i(16)
+    end
+
+    def guess_gem(path)
+      @gems[path] ||=
+        if %r{(/gems/.*)*/gems/(?<gemname>[^/]+)} =~ path
+          gemname
+        elsif %r{/rubygems[\./]}.match?(path)
+          "rubygems"
+        elsif %r{ruby/2\.[^/]+/(?<stdlib>[^/\.]+)} =~ path
+          stdlib
+        elsif %r{(?<app>[^/]+/(bin|app|lib))} =~ path
+          app
+        else
+          "other"
+        end
+    end
+  end
+end