heap-profiler 0.8.0.rc1-aarch64-linux

data/lib/heap_profiler/parser.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  module Parser
+    CLASS_DEFAULT_PROC = ->(_hash, key) { "<Class#0x#{key.to_s(16)}>" }
+
+    class << self
+      attr_accessor :batch_size
+    end
+    self.batch_size = 10_000_000 # 10MB
+
+    class Ruby
+      def build_index(path)
+        require 'json'
+        classes_index = {}
+        classes_index.default_proc = CLASS_DEFAULT_PROC
+        strings_index = {}
+
+        File.open(path).each_line do |line|
+          object = JSON.parse(line, symbolize_names: true)
+          case object[:type]
+          when 'MODULE', 'CLASS'
+            address = parse_address(object[:address])
+
+            name = object[:name]
+            name ||= if object[:file] && object[:line]
+              "<Class #{object[:file]}:#{object[:line]}>"
+            end
+
+            if name
+              classes_index[address] = name
+            end
+          when 'STRING'
+            next if object[:shared]
+            if (value = object[:value])
+              strings_index[parse_address(object[:address])] = value
+            end
+          end
+        end
+
+        [classes_index, strings_index]
+      end
+
+      def parse_address(address)
+        address.to_i(16)
+      end
+    end
+
+    class Native
+      def build_index(path, batch_size: Parser.batch_size)
+        indexes = _build_index(path, batch_size)
+        indexes.first.default_proc = CLASS_DEFAULT_PROC
+        indexes
+      end
+
+      def load_many(path, since: nil, batch_size: Parser.batch_size, &block)
+        _load_many(path, since, batch_size, &block)
+      end
+    end
+
+    class << self
+      def build_index(path)
+        current.build_index(path)
+      end
+
+      def load_many(path, **kwargs, &block)
+        current.load_many(path, **kwargs, &block)
+      end
+
+      private
+
+      def current
+        Thread.current[:HeapProfilerParser] ||= Native.new
+      end
+    end
+  end
+
+  begin
+    require "heap_profiler/#{RUBY_VERSION[/^\d+\.\d+/]}/heap_profiler"
+  rescue LoadError
+    require "heap_profiler/heap_profiler"
+  end
+end

data/lib/heap_profiler/polychrome.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  module Polychrome
+    class << self
+      def path(text)
+        blue(text)
+      end
+
+      def string(text)
+        green(text)
+      end
+
+      def line(text)
+        cyan(text)
+      end
+
+      private
+
+      def black(str)
+        "\033[30m#{str}\033[0m"
+      end
+
+      def red(str)
+        "\033[31m#{str}\033[0m"
+      end
+
+      def green(str)
+        "\033[32m#{str}\033[0m"
+      end
+
+      def brown(str)
+        "\033[33m#{str}\033[0m"
+      end
+
+      def blue(str)
+        "\033[34m#{str}\033[0m"
+      end
+
+      def magenta(str)
+        "\033[35m#{str}\033[0m"
+      end
+
+      def cyan(str)
+        "\033[36m#{str}\033[0m"
+      end
+
+      def gray(str)
+        "\033[37m#{str}\033[0m"
+      end
+
+      def bg_black(str)
+        "\033[40m#{str}\033[0m"
+      end
+
+      def bg_red(str)
+        "\033[41m#{str}\033[0m"
+      end
+
+      def bg_green(str)
+        "\033[42m#{str}\033[0m"
+      end
+
+      def bg_brown(str)
+        "\033[43m#{str}\033[0m"
+      end
+
+      def bg_blue(str)
+        "\033[44m#{str}\033[0m"
+      end
+
+      def bg_magenta(str)
+        "\033[45m#{str}\033[0m"
+      end
+
+      def bg_cyan(str)
+        "\033[46m#{str}\033[0m"
+      end
+
+      def bg_gray(str)
+        "\033[47m#{str}\033[0m"
+      end
+
+      def bold(str)
+        "\033[1m#{str}\033[22m"
+      end
+
+      def reverse_color(str)
+        "\033[7m#{str}\033[27m"
+      end
+    end
+  end
+end

data/lib/heap_profiler/reporter.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class << self
+    # This works around a Ruby bug present until at least 2.7.1
+    # ObjectSpace.dump include module and class names in the dump
+    # and for anonymous modules and classes this mean naming them.
+    #
+    # So we name them at the start of the profile to avoid that.
+    #
+    # See: https://github.com/ruby/ruby/pull/3349
+    if RUBY_VERSION < '2.8'
+      def name_anonymous_modules!
+        ObjectSpace.each_object(Module) do |mod|
+          next if mod.singleton_class?
+          next if real_mod_name(mod)
+          # We have to assign it at the top level to avoid allocating a string for the name
+          ::Object.const_set(:AnonymousClassOrModule, mod)
+          ::Object.send(:remove_const, :AnonymousClassOrModule)
+        end
+      end
+
+      ::Module.alias_method(:__real_mod_name, :name)
+      def real_mod_name(mod)
+        mod.__real_mod_name
+      end
+    else
+      def name_anonymous_modules!
+      end
+    end
+  end
+
+  class Reporter
+    def initialize(dir_path)
+      @dir_path = dir_path
+      @enable_tracing = !allocation_tracing_enabled?
+      @generation = nil
+      @partial = true
+    end
+
+    def start(partial: true)
+      @partial = partial
+      FileUtils.mkdir_p(@dir_path)
+      ObjectSpace.trace_object_allocations_start if @enable_tracing
+
+      @allocated_heap = open_heap("allocated")
+      @retained_heap = open_heap("retained")
+
+      HeapProfiler.name_anonymous_modules!
+
+      GC.start
+      GC.disable
+      @generation = GC.count
+    end
+
+    def stop
+      HeapProfiler.name_anonymous_modules!
+      ObjectSpace.trace_object_allocations_stop if @enable_tracing
+
+      # we can't use partial dump for allocated.heap, because we need old generations
+      # as well to build the classes and strings indexes.
+      dump_heap(@allocated_heap)
+
+      GC.enable
+      GC.start
+      dump_heap(@retained_heap, partial: @partial)
+      @allocated_heap.close
+      @retained_heap.close
+      write_info("generation", @partial ? @generation.to_s : "0")
+    end
+
+    def run
+      start
+      begin
+        yield
+      rescue Exception
+        ObjectSpace.trace_object_allocations_stop if @enable_tracing
+        GC.enable
+        raise
+      else
+        stop
+      end
+    end
+
+    private
+
+    def write_info(key, value)
+      File.write(File.join(@dir_path, "#{key}.info"), value)
+    end
+
+    if RUBY_VERSION >= '3.0'
+      def dump_heap(file, partial: false)
+        ObjectSpace.dump_all(output: file, since: partial ? @generation : nil)
+        file.close
+      end
+    else
+      # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
+      # because of this even en empty block of code will report a handful of allocations.
+      # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
+      # property.
+      class_eval <<~RUBY, '__hprof', __LINE__
+        # frozen_string_literal: true
+        def dump_heap(file, partial: false)
+          ObjectSpace.dump_all(output: file)
+          file.close
+        end
+      RUBY
+    end
+
+    def open_heap(name)
+      File.open(File.join(@dir_path, "#{name}.heap"), 'w+')
+    end
+
+    def allocation_tracing_enabled?
+      ObjectSpace.allocation_sourceline(Object.new)
+    end
+  end
+end

data/lib/heap_profiler/results.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+module HeapProfiler
+  class AbstractResults
+    UNIT_PREFIXES = {
+      0 => 'B',
+      3 => 'kB',
+      6 => 'MB',
+      9 => 'GB',
+      12 => 'TB',
+      15 => 'PB',
+      18 => 'EB',
+      21 => 'ZB',
+      24 => 'YB',
+    }.freeze
+
+    METRICS = ["memory", "objects", "strings", "shape_edges"].freeze
+    GROUPED_METRICS = ["memory", "objects"]
+    GROUPINGS = ["gem", "file", "location", "class"].freeze
+
+    attr_reader :types, :dimensions
+
+    @top_entries_count = 50
+    class << self
+      attr_accessor :top_entries_count
+    end
+
+    def initialize(*, **)
+      raise NotImplementedError
+    end
+
+    def print_title(io, title)
+      io.puts
+      io.puts title
+      io.puts @colorize.line("-----------------------------------")
+    end
+
+    def print_output(io, topic, detail)
+      io.puts "#{@colorize.path(topic.to_s.rjust(10))}  #{detail}"
+    end
+
+    def print_output2(io, topic1, topic2, detail)
+      io.puts "#{@colorize.path(topic1.to_s.rjust(10))}  #{@colorize.path(topic2.to_s.rjust(6))}  #{detail}"
+    end
+
+    def normalize_path(path)
+      @normalize_path ||= {}
+      @normalize_path[path] ||= begin
+        if %r!(/gems/.*)*/gems/(?<gemname>[^/]+)(?<rest>.*)! =~ path
+          "#{gemname}#{rest}"
+        elsif %r!ruby/2\.[^/]+/(?<stdlib>[^/.]+)(?<rest>.*)! =~ path
+          "ruby/lib/#{stdlib}#{rest}"
+        elsif %r!(?<app>[^/]+/(bin|app|lib))(?<rest>.*)! =~ path
+          "#{app}#{rest}"
+        else
+          path
+        end
+      end
+    end
+
+    def scale_bytes(bytes)
+      return "0 B" if bytes.zero?
+
+      scale = Math.log10(bytes).div(3) * 3
+      scale = 24 if scale > 24
+      format("%.2f #{UNIT_PREFIXES[scale]}", bytes / 10.0**scale)
+    end
+  end
+
+  class HeapResults < AbstractResults
+    def initialize(heap_path, metrics = METRICS, groupings = GROUPINGS)
+      @path = heap_path
+      @metrics = metrics
+      @groupings = groupings
+    end
+
+    def pretty_print(io = $stdout, **options)
+      heap = Dump.new(@path)
+      index = Index.new(heap)
+
+      color_output = options.fetch(:color_output) { io.respond_to?(:isatty) && io.isatty }
+      @colorize = color_output ? Polychrome : Monochrome
+
+      analyzer = Analyzer.new(heap, index)
+      dimensions = analyzer.run(@metrics, @groupings)
+
+      if dimensions['total']
+        io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
+                "(#{dimensions['total'].objects} objects)"
+      end
+
+      @metrics.each do |metric|
+        next unless GROUPED_METRICS.include?(metric)
+        @groupings.each do |grouping|
+          dump_data(io, dimensions, metric, grouping, options)
+        end
+      end
+
+      if @metrics.include?("strings")
+        dump_strings(io, dimensions, options)
+      end
+
+      if @metrics.include?("shape_edges")
+        dump_shape_edges(io, dimensions, options)
+      end
+    end
+
+    def dump_data(io, dimensions, metric, grouping, options)
+      print_title io, "#{metric} by #{grouping}"
+      data = dimensions[grouping].top_n(metric, AbstractResults.top_entries_count)
+
+      scale_data = metric == "memory" && options[:scale_bytes]
+      normalize_paths = options[:normalize_paths]
+
+      if data && !data.empty?
+        data.each { |pair| pair[0] = normalize_path(pair[0]) } if normalize_paths
+        data.each { |pair| pair[1] = scale_bytes(pair[1]) } if scale_data
+        data.each { |k, v| print_output(io, v, k) }
+      else
+        io.puts "NO DATA"
+      end
+    end
+
+    def dump_strings(io, dimensions, options)
+      normalize_paths = options[:normalize_paths]
+      scale_data = options[:scale_bytes]
+      top = AbstractResults.top_entries_count
+
+      print_title(io, "String Report")
+
+      dimensions["strings"].top_n(top).each do |string|
+        memsize = scale_data ? scale_bytes(string.memsize) : string.memsize
+        print_output2 io, memsize, string.count, @colorize.string(string.value.inspect)
+        string.top_n(top).each do |string_location|
+          location = string_location.location
+          location = normalize_path(location) if normalize_paths
+          print_output2 io, '', string_location.count, location
+        end
+        io.puts
+      end
+    end
+
+    def dump_shape_edges(io, dimensions, _options)
+      top = AbstractResults.top_entries_count
+
+      data = dimensions["shape_edges"].top_n(top)
+      unless data.empty?
+        print_title(io, "Shape Edges Report")
+
+        data.each do |edge_name, count|
+          print_output io, count, edge_name
+        end
+      end
+    end
+  end
+
+  class DiffResults < AbstractResults
+    TYPES = ["allocated", "retained"].freeze
+
+    def initialize(directory, types = TYPES, metrics = METRICS, groupings = GROUPINGS)
+      @directory = directory
+      @types = types
+      @metrics = metrics
+      @groupings = groupings
+    end
+
+    def pretty_print(io = $stdout, **options)
+      diff = Diff.new(@directory)
+      heaps = @types.each_with_object({}) { |t, h| h[t] = diff.public_send("#{t}_diff") }
+      index = Index.new(diff.allocated)
+
+      color_output = options.fetch(:color_output) { io.respond_to?(:isatty) && io.isatty }
+      @colorize = color_output ? Polychrome : Monochrome
+
+      dimensions = {}
+      heaps.each do |type, heap|
+        analyzer = Analyzer.new(heap, index)
+        dimensions[type] = analyzer.run(@metrics, @groupings)
+      end
+
+      dimensions.each do |type, metrics|
+        io.puts "Total #{type}: #{scale_bytes(metrics['total'].memory)} " \
+                "(#{metrics['total'].objects} objects)"
+      end
+
+      @types.each do |type|
+        @metrics.each do |metric|
+          next unless GROUPED_METRICS.include?(metric)
+          @groupings.each do |grouping|
+            dump_data(io, dimensions, type, metric, grouping, options)
+          end
+        end
+      end
+
+      if @metrics.include?("strings")
+        @types.each do |type|
+          dump_strings(io, dimensions[type], type, options)
+        end
+      end
+
+      if @metrics.include?("shape_edges")
+        @types.each do |type|
+          dump_shape_edges(io, dimensions[type], type, options)
+        end
+      end
+    end
+
+    def dump_data(io, dimensions, type, metric, grouping, options)
+      print_title io, "#{type} #{metric} by #{grouping}"
+      data = dimensions[type][grouping].top_n(metric, AbstractResults.top_entries_count)
+
+      scale_data = metric == "memory" && options[:scale_bytes]
+      normalize_paths = options[:normalize_paths]
+
+      if data && !data.empty?
+        data.each { |pair| pair[0] = normalize_path(pair[0]) } if normalize_paths
+        data.each { |pair| pair[1] = scale_bytes(pair[1]) } if scale_data
+        data.each { |k, v| print_output(io, v, k) }
+      else
+        io.puts "NO DATA"
+      end
+    end
+
+    def dump_strings(io, dimensions, type, options)
+      normalize_paths = options[:normalize_paths]
+      scale_data = options[:scale_bytes]
+      top = AbstractResults.top_entries_count
+
+      print_title(io, "#{type.capitalize} String Report")
+
+      dimensions["strings"].top_n(top).each do |string|
+        memsize = scale_data ? scale_bytes(string.memsize) : string.memsize
+        print_output2 io, memsize, string.count, @colorize.string(string.value.inspect)
+        string.top_n(top).each do |string_location|
+          location = string_location.location
+          location = normalize_path(location) if normalize_paths
+          print_output2 io, '', string_location.count, location
+        end
+        io.puts
+      end
+    end
+
+    def dump_shape_edges(io, dimensions, _type, _options)
+      top = AbstractResults.top_entries_count
+
+      data = dimensions["shape_edges"].top_n(top)
+      unless data.empty?
+        print_title(io, "Shape Edges Report")
+
+        data.each do |edge_name, count|
+          print_output io, count, edge_name
+        end
+      end
+    end
+  end
+end

data/lib/heap_profiler/runtime.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "objspace"
+require "fileutils"
+
+require "heap_profiler/version"
+require "heap_profiler/reporter"
+
+module HeapProfiler
+  Error = Class.new(StandardError)
+  CapacityError = Class.new(Error)
+
+  class << self
+    attr_accessor :current_reporter
+
+    def start(dir, **kwargs)
+      return if current_reporter
+      self.current_reporter = Reporter.new(dir)
+      current_reporter.start(**kwargs)
+    end
+
+    def stop
+      current_reporter&.stop
+    end
+
+    def report(dir, &block)
+      Reporter.new(dir).run(&block)
+    end
+  end
+end

data/lib/heap_profiler/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module HeapProfiler
+  VERSION = "0.8.0.rc1"
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: heap-profiler
+version: !ruby/object:Gem::Version
+  version: 0.8.0.rc1
+platform: aarch64-linux
+authors:
+- Jean Boussier
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2026-01-14 00:00:00.000000000 Z
+dependencies: []
+description: Make several heap dumps and summarize allocated, retained memory
+email:
+- jean.boussier@gmail.com
+executables:
+- heap-profiler
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/cibuildgem.yaml"
+- ".github/workflows/tests.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO.md
+- benchmark/address-parsing.rb
+- benchmark/indexing.rb
+- bin/console
+- bin/generate-report
+- bin/rubocop
+- bin/setup
+- bin/testunit
+- dev.yml
+- exe/heap-profiler
+- ext/heap_profiler/extconf.rb
+- ext/heap_profiler/heap_profiler.cpp
+- ext/heap_profiler/simdjson.cpp
+- ext/heap_profiler/simdjson.h
+- heap-profiler.gemspec
+- lib/heap-profiler.rb
+- lib/heap_profiler/3.1/heap_profiler.so
+- lib/heap_profiler/3.2/heap_profiler.so
+- lib/heap_profiler/3.3/heap_profiler.so
+- lib/heap_profiler/3.4/heap_profiler.so
+- lib/heap_profiler/4.0/heap_profiler.so
+- lib/heap_profiler/analyzer.rb
+- lib/heap_profiler/cli.rb
+- lib/heap_profiler/diff.rb
+- lib/heap_profiler/dump.rb
+- lib/heap_profiler/full.rb
+- lib/heap_profiler/index.rb
+- lib/heap_profiler/monochrome.rb
+- lib/heap_profiler/parser.rb
+- lib/heap_profiler/polychrome.rb
+- lib/heap_profiler/reporter.rb
+- lib/heap_profiler/results.rb
+- lib/heap_profiler/runtime.rb
+- lib/heap_profiler/version.rb
+homepage: https://github.com/Shopify/heap-profiler
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org/
+  homepage_uri: https://github.com/Shopify/heap-profiler
+  source_code_uri: https://github.com/Shopify/heap-profiler
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 4.1.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.3.27
+signing_key: 
+specification_version: 4
+summary: Ruby heap profiling tool
+test_files: []