heap-profiler 0.2.0 → 0.5.0

data/heap-profiler.gemspec

@@ -3,7 +3,7 @@ require_relative 'lib/heap_profiler/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "heap-profiler"
-  spec.version       = Heap::Profiler::VERSION
+  spec.version       = HeapProfiler::VERSION
   spec.authors       = ["Jean Boussier"]
   spec.email         = ["jean.boussier@gmail.com"]
 

data/lib/heap_profiler/analyzer.rb

@@ -11,9 +11,6 @@ module HeapProfiler
 
       def process(_index, object)
         @objects += 1
-        unless object[:memsize]
-          p object
-        end
         @memory += object[:memsize]
       end
 

data/lib/heap_profiler/cli.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'optparse'
 
 module HeapProfiler
   class CLI
@@ -7,13 +8,21 @@ module HeapProfiler
     end
 
     def run
-      if @argv.size == 1
-        print_report(@argv.first)
-        0
-      else
-        print_usage
-        1
+      parser.parse!(@argv)
+
+      begin
+        if @argv.size == 1
+          print_report(@argv.first)
+          return 0
+        end
+      rescue CapacityError => error
+        STDERR.puts(error.message)
+        STDERR.puts("Current size: #{Parser.batch_size}B")
+        STDERR.puts("Try increasing it with --batch-size")
+        STDERR.puts
       end
+      print_usage
+      1
     end
 
     def print_report(path)
@@ -28,5 +37,51 @@ module HeapProfiler
     def print_usage
       puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
     end
+
+    SIZE_UNITS = {
+      'B' => 1,
+      'K' => 1_000,
+      'M' => 1_000_000,
+      'G' => 1_000_000_000,
+    }
+    def parse_byte_size(size_string)
+      if (match = size_string.match(/\A(\d+)(\w)?B?\z/i))
+        digits = Float(match[1])
+        base = 1
+        unit = match[2]&.upcase
+        if unit
+          base = SIZE_UNITS.fetch(unit) { raise ArgumentError, "Unknown size unit: #{unit}" }
+        end
+        size = (digits * base).to_i
+        if size > 4_000_000_000
+          raise ArgumentError, "Batch size can't be bigger than 4G"
+        end
+        size
+      else
+        raise ArgumentError, "#{size_string} is not a valid size"
+      end
+    end
+
+    private
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: heap-profiler [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~EOS
+          Sets the simdjson parser batch size. It must be larger than the largest JSON document in the
+          heap dump, and defaults to 10MB.
+        EOS
+        opts.on('--batch-size SIZE', help.strip) do |size_string|
+          HeapProfiler::Parser.batch_size = parse_byte_size(size_string)
+        rescue ArgumentError => error
+          STDERR.puts "Invalid batch-size: #{error.message}"
+          exit 1
+        end
+      end
+    end
   end
 end

data/lib/heap_profiler/dump.rb

@@ -61,8 +61,8 @@ module HeapProfiler
       end
     end
 
-    def each_object(since: 0, &block)
-      Parser.load_many(path, since: since, batch_size: 10_000_000, &block)
+    def each_object(since: nil, &block)
+      Parser.load_many(path, since: since, &block)
     end
 
     def stats

data/lib/heap_profiler/index.rb

@@ -34,7 +34,7 @@ module HeapProfiler
     }.freeze
 
     IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }
-    DATA_TYPES = Hash.new { |h, k| h[k] = "<#{(k || 'unknown')}> (DATA)" }
+    DATA_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (DATA)" }
 
     def guess_class(object)
       type = object[:type]
@@ -44,14 +44,19 @@ module HeapProfiler
 
       return IMEMO_TYPES[object[:imemo_type]] if type == :IMEMO
 
-      class_address = object[:class]
-      return unless class_address
+      class_name = if (class_address = object[:class])
+        @classes.fetch(class_address) do
+          return DATA_TYPES[object[:struct]] if type == :DATA
 
-      @classes.fetch(class_address) do
-        return DATA_TYPES[object[:struct]] if type == :DATA
+          $stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
+          nil
+        end
+      end
 
-        $stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
-        nil
+      if type == :DATA && (class_name.nil? || class_name == "Object")
+        DATA_TYPES[object[:struct]]
+      else
+        class_name
       end
     end
 

data/lib/heap_profiler/parser.rb

@@ -2,18 +2,33 @@
 
 module HeapProfiler
   module Parser
+    CLASS_DEFAULT_PROC = ->(_hash, key) { "<Class#0x#{key.to_s(16)}>" }
+
+    class << self
+      attr_accessor :batch_size
+    end
+    self.batch_size = 10_000_000 # 10MB
+
     class Ruby
       def build_index(path)
         require 'json'
         classes_index = {}
+        classes_index.default_proc = CLASS_DEFAULT_PROC
         strings_index = {}
 
         File.open(path).each_line do |line|
           object = JSON.parse(line, symbolize_names: true)
           case object[:type]
           when 'MODULE', 'CLASS'
-            if (name = object[:name])
-              classes_index[parse_address(object[:address])] = name
+            address = parse_address(object[:address])
+
+            name = object[:name]
+            name ||= if object[:file] && object[:line]
+              "<Class #{object[:file]}:#{object[:line]}>"
+            end
+
+            if name
+              classes_index[address] = name
             end
           when 'STRING'
             next if object[:shared]
@@ -32,13 +47,13 @@ module HeapProfiler
     end
 
     class Native
-      DEFAULT_BATCH_SIZE = 10_000_000 # 10MB
-
-      def build_index(path, batch_size: DEFAULT_BATCH_SIZE)
-        _build_index(path, batch_size)
+      def build_index(path, batch_size: Parser.batch_size)
+        indexes = _build_index(path, batch_size)
+        indexes.first.default_proc = CLASS_DEFAULT_PROC
+        indexes
       end
 
-      def load_many(path, since: nil, batch_size: DEFAULT_BATCH_SIZE, &block)
+      def load_many(path, since: nil, batch_size: Parser.batch_size, &block)
         _load_many(path, since, batch_size, &block)
       end
     end

data/lib/heap_profiler/reporter.rb

@@ -9,8 +9,6 @@ module HeapProfiler
     # So we name them at the start of the profile to avoid that.
     #
     # See: https://github.com/ruby/ruby/pull/3349
-    #
-    # TODO: Could we actually do the dump ourselves? objspace is a extension already.
     if RUBY_VERSION < '2.8'
       def name_anonymous_modules!
         ObjectSpace.each_object(Module) do |mod|
@@ -56,10 +54,14 @@ module HeapProfiler
     def stop
       HeapProfiler.name_anonymous_modules!
       ObjectSpace.trace_object_allocations_stop if @enable_tracing
+
+      # we can't use partial dump for allocated.heap, because we need old generations
+      # as well to build the classes and strings indexes.
       dump_heap(@allocated_heap)
+
       GC.enable
       4.times { GC.start }
-      dump_heap(@retained_heap)
+      dump_heap(@retained_heap, partial: true)
       @allocated_heap.close
       @retained_heap.close
       write_info("generation", @generation.to_s)
@@ -84,17 +86,24 @@ module HeapProfiler
       File.write(File.join(@dir_path, "#{key}.info"), value)
     end
 
-    # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
-    # because of this even en empty block of code will report a handful of allocations.
-    # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
-    # property.
-    class_eval <<~RUBY, '__hprof', __LINE__
-      # frozen_string_literal: true
-      def dump_heap(file)
-        ObjectSpace.dump_all(output: file)
+    if RUBY_VERSION >= '3.0'
+      def dump_heap(file, partial: false)
+        ObjectSpace.dump_all(output: file, since: partial ? @generation : nil)
         file.close
       end
-    RUBY
+    else
+      # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
+      # because of this even en empty block of code will report a handful of allocations.
+      # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
+      # property.
+      class_eval <<~RUBY, '__hprof', __LINE__
+        # frozen_string_literal: true
+        def dump_heap(file, partial: false)
+          ObjectSpace.dump_all(output: file)
+          file.close
+        end
+      RUBY
+    end
 
     def open_heap(name)
       File.open(File.join(@dir_path, "#{name}.heap"), 'w+')

data/lib/heap_profiler/runtime.rb

@@ -8,6 +8,7 @@ require "heap_profiler/reporter"
 
 module HeapProfiler
   Error = Class.new(StandardError)
+  CapacityError = Class.new(Error)
 
   class << self
     attr_accessor :current_reporter

data/lib/heap_profiler/version.rb

@@ -1,6 +1,4 @@
 # frozen_string_literal: true
-module Heap
-  module Profiler
-    VERSION = "0.2.0"
-  end
+module HeapProfiler
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heap-profiler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-19 00:00:00.000000000 Z
+date: 2021-08-05 00:00:00.000000000 Z
 dependencies: []
 description: Make several heap dumps and summarize allocated, retained memory
 email:
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.20
 signing_key:
 specification_version: 4
 summary: Ruby heap profiling tool