heap-profiler 0.5.0 → 0.7.0

data/lib/heap_profiler/analyzer.rb

@@ -143,12 +143,13 @@ module HeapProfiler
           end
         end
 
-        def top_n(_max)
+        def top_n(max)
           values = @locations_counts.values
           values.sort! do |a, b|
             cmp = b.count <=> a.count
             cmp == 0 ? b.location <=> a.location : cmp
           end
+          values.take(max)
         end
       end
 
@@ -178,6 +179,29 @@ module HeapProfiler
       end
     end
 
+    class ShapeEdgeDimension
+      def initialize
+        @stats = Hash.new(0)
+      end
+
+      def process(_index, object)
+        if name = object[:edge_name]
+          @stats[name] += 1
+        end
+      end
+
+      def top_n(max)
+        @stats.sort do |(a_name, a_count), (b_name, b_count)|
+          cmp = b_count <=> a_count
+          if cmp == 0
+            a_name <=> b_name
+          else
+            cmp
+          end
+        end.take(max)
+      end
+    end
+
     def initialize(heap, index)
       @heap = heap
       @index = index
@@ -188,6 +212,8 @@ module HeapProfiler
       metrics.each do |metric|
         if metric == "strings"
           dimensions["strings"] = StringDimension.new
+        elsif metric == "shape_edges"
+          dimensions["shape_edges"] = ShapeEdgeDimension.new
         else
           dimensions["total"] = Dimension.new
           groupings.each do |grouping|

data/lib/heap_profiler/cli.rb

@@ -11,9 +11,18 @@ module HeapProfiler
       parser.parse!(@argv)
 
       begin
-        if @argv.size == 1
-          print_report(@argv.first)
+        case @argv.first
+        when "clean"
+          clean_dump(@argv[1])
           return 0
+        when "report"
+          print_report(@argv[1])
+          return 0
+        else
+          if @argv.size == 1
+            print_report(@argv.first)
+            return 0
+          end
         end
       rescue CapacityError => error
         STDERR.puts(error.message)
@@ -27,15 +36,43 @@ module HeapProfiler
 
     def print_report(path)
       results = if File.directory?(path)
-        DiffResults.new(path)
+        if @retained_only
+          DiffResults.new(path, ["retained"])
+        else
+          DiffResults.new(path)
+        end
       else
         HeapResults.new(path)
       end
       results.pretty_print(scale_bytes: true, normalize_paths: true)
     end
 
+    def clean_dump(path)
+      require "json"
+      errors = index = 0
+      clean_path = "#{path}.clean"
+      File.open(clean_path, "w+") do |output|
+        File.open(path) do |input|
+          input.each_line do |line|
+            begin
+              JSON.parse(line)
+            rescue JSON::ParserError
+              errors += 1
+              $stderr.puts("Invalid JSON found on line #{index}. Skipping")
+            else
+              output.print(line)
+            end
+            index += 1
+          end
+        end
+      end
+      $stderr.puts("Processed #{index} lines, removed #{errors} invalid lines")
+      $stderr.puts("Clean dump available at #{clean_path}")
+    end
+
     def print_usage
       puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
+      puts @parser.help
     end
 
     SIZE_UNITS = {
@@ -66,14 +103,30 @@ module HeapProfiler
 
     def parser
       @parser ||= OptionParser.new do |opts|
-        opts.banner = "Usage: heap-profiler [ARGS]"
-        opts.separator ""
-        opts.separator "GLOBAL OPTIONS"
+        opts.banner = <<~EOS
+          Usage: heap-profiler [SUBCOMMAND] [ARGS]"
+
+          SUBCOMMANDS
+
+            report: Produce a full memory report from the provided dump. (default)
+
+            clean: Remove all malformed lines from the provided heap dump. Can be useful to workaround some ruby bugs.
+
+          GLOBAL OPTIONS
+        EOS
         opts.separator ""
 
-        help = <<~EOS
-          Sets the simdjson parser batch size. It must be larger than the largest JSON document in the
-          heap dump, and defaults to 10MB.
+        opts.on('-r', '--retained-only', 'Only compute report for memory retentions.') do
+          @retained_only = true
+        end
+
+        HeapProfiler::AbstractResults.top_entries_count = 50
+        opts.on("-m", "--max=NUM", Integer, "Max number of entries to output. (Defaults to 50)") do |arg|
+          HeapProfiler::AbstractResults.top_entries_count = arg
+        end
+
+        help = <<~EOS.lines.join(" ")
+          Sets the simdjson parser batch size. It must be larger than the largest JSON document in the heap dump, and defaults to 10MB.
         EOS
         opts.on('--batch-size SIZE', help.strip) do |size_string|
           HeapProfiler::Parser.batch_size = parse_byte_size(size_string)

data/lib/heap_profiler/diff.rb

@@ -19,6 +19,7 @@ module HeapProfiler
       @report_directory = report_directory
       @allocated = open_dump('allocated')
       @generation = Integer(File.read(File.join(report_directory, 'generation.info')))
+      @generation = nil if @generation == 0
     end
 
     def allocated_diff

data/lib/heap_profiler/index.rb

@@ -31,6 +31,7 @@ module HeapProfiler
       REGEXP: "Regexp",
       MATCH: "MatchData",
       ROOT: "<VM Root>",
+      SHAPE: "SHAPE",
     }.freeze
 
     IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }

data/lib/heap_profiler/reporter.rb

@@ -35,9 +35,11 @@ module HeapProfiler
       @dir_path = dir_path
       @enable_tracing = !allocation_tracing_enabled?
       @generation = nil
+      @partial = true
     end
 
-    def start
+    def start(partial: true)
+      @partial = partial
       FileUtils.mkdir_p(@dir_path)
       ObjectSpace.trace_object_allocations_start if @enable_tracing
 
@@ -46,7 +48,7 @@ module HeapProfiler
 
       HeapProfiler.name_anonymous_modules!
 
-      4.times { GC.start }
+      GC.start
       GC.disable
       @generation = GC.count
     end
@@ -60,11 +62,11 @@ module HeapProfiler
       dump_heap(@allocated_heap)
 
       GC.enable
-      4.times { GC.start }
-      dump_heap(@retained_heap, partial: true)
+      GC.start
+      dump_heap(@retained_heap, partial: @partial)
       @allocated_heap.close
       @retained_heap.close
-      write_info("generation", @generation.to_s)
+      write_info("generation", @partial ? @generation.to_s : "0")
     end
 
     def run

data/lib/heap_profiler/results.rb

@@ -14,11 +14,17 @@ module HeapProfiler
       24 => 'YB',
     }.freeze
 
-    METRICS = ["memory", "objects", "strings"].freeze
+    METRICS = ["memory", "objects", "strings", "shape_edges"].freeze
+    GROUPED_METRICS = ["memory", "objects"]
     GROUPINGS = ["gem", "file", "location", "class"].freeze
 
     attr_reader :types, :dimensions
 
+    @top_entries_count = 50
+    class << self
+      attr_accessor :top_entries_count
+    end
+
     def initialize(*, **)
       raise NotImplementedError
     end
@@ -78,11 +84,13 @@ module HeapProfiler
       analyzer = Analyzer.new(heap, index)
       dimensions = analyzer.run(@metrics, @groupings)
 
-      io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
-              "(#{dimensions['total'].objects} objects)"
+      if dimensions['total']
+        io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
+                "(#{dimensions['total'].objects} objects)"
+      end
 
       @metrics.each do |metric|
-        next if metric == "strings"
+        next unless GROUPED_METRICS.include?(metric)
         @groupings.each do |grouping|
           dump_data(io, dimensions, metric, grouping, options)
         end
@@ -91,11 +99,15 @@ module HeapProfiler
       if @metrics.include?("strings")
         dump_strings(io, dimensions, options)
       end
+
+      if @metrics.include?("shape_edges")
+        dump_shape_edges(io, dimensions, options)
+      end
     end
 
     def dump_data(io, dimensions, metric, grouping, options)
       print_title io, "#{metric} by #{grouping}"
-      data = dimensions[grouping].top_n(metric, options.fetch(:top, 50))
+      data = dimensions[grouping].top_n(metric, AbstractResults.top_entries_count)
 
       scale_data = metric == "memory" && options[:scale_bytes]
       normalize_paths = options[:normalize_paths]
@@ -112,7 +124,7 @@ module HeapProfiler
     def dump_strings(io, dimensions, options)
       normalize_paths = options[:normalize_paths]
       scale_data = options[:scale_bytes]
-      top = options.fetch(:top, 50)
+      top = AbstractResults.top_entries_count
 
       print_title(io, "String Report")
 
@@ -127,6 +139,19 @@ module HeapProfiler
         io.puts
       end
     end
+
+    def dump_shape_edges(io, dimensions, _options)
+      top = AbstractResults.top_entries_count
+
+      data = dimensions["shape_edges"].top_n(top)
+      unless data.empty?
+        print_title(io, "Shape Edges Report")
+
+        data.each do |edge_name, count|
+          print_output io, count, edge_name
+        end
+      end
+    end
   end
 
   class DiffResults < AbstractResults
@@ -160,7 +185,7 @@ module HeapProfiler
 
       @types.each do |type|
         @metrics.each do |metric|
-          next if metric == "strings"
+          next unless GROUPED_METRICS.include?(metric)
           @groupings.each do |grouping|
             dump_data(io, dimensions, type, metric, grouping, options)
           end
@@ -172,11 +197,17 @@ module HeapProfiler
           dump_strings(io, dimensions[type], type, options)
         end
       end
+
+      if @metrics.include?("shape_edges")
+        @types.each do |type|
+          dump_shape_edges(io, dimensions[type], type, options)
+        end
+      end
     end
 
     def dump_data(io, dimensions, type, metric, grouping, options)
       print_title io, "#{type} #{metric} by #{grouping}"
-      data = dimensions[type][grouping].top_n(metric, options.fetch(:top, 50))
+      data = dimensions[type][grouping].top_n(metric, AbstractResults.top_entries_count)
 
       scale_data = metric == "memory" && options[:scale_bytes]
       normalize_paths = options[:normalize_paths]
@@ -193,7 +224,7 @@ module HeapProfiler
     def dump_strings(io, dimensions, type, options)
       normalize_paths = options[:normalize_paths]
       scale_data = options[:scale_bytes]
-      top = options.fetch(:top, 50)
+      top = AbstractResults.top_entries_count
 
       print_title(io, "#{type.capitalize} String Report")
 
@@ -208,5 +239,18 @@ module HeapProfiler
         io.puts
       end
     end
+
+    def dump_shape_edges(io, dimensions, _type, _options)
+      top = AbstractResults.top_entries_count
+
+      data = dimensions["shape_edges"].top_n(top)
+      unless data.empty?
+        print_title(io, "Shape Edges Report")
+
+        data.each do |edge_name, count|
+          print_output io, count, edge_name
+        end
+      end
+    end
   end
 end

data/lib/heap_profiler/runtime.rb

@@ -13,10 +13,10 @@ module HeapProfiler
   class << self
     attr_accessor :current_reporter
 
-    def start(dir)
+    def start(dir, **kwargs)
       return if current_reporter
       self.current_reporter = Reporter.new(dir)
-      current_reporter.start
+      current_reporter.start(**kwargs)
     end
 
     def stop

data/lib/heap_profiler/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module HeapProfiler
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heap-profiler
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-05 00:00:00.000000000 Z
+date: 2023-02-27 00:00:00.000000000 Z
 dependencies: []
 description: Make several heap dumps and summarize allocated, retained memory
 email:
@@ -22,7 +22,6 @@ files:
 - ".github/workflows/tests.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -79,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: Ruby heap profiling tool

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.3
-before_install: gem install bundler -v 2.1.4