heap-profiler 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1e0a1ab7e684de76d217df0eef23331ab4ab32a2b252766999db26ce73670c2
-  data.tar.gz: 50209c2943414c65167307a67e9454807619e8a9a43cbf74c9a09933c2eabba3
+  metadata.gz: e84c345fb6a29dbb5da52ce051ae7ca87b220adaea020458b4fc44f9343ad2d2
+  data.tar.gz: ca8aa6b5a67209444ece911c598ff4d0fc075bc68274a2507273bf1f6b35ee8c
 SHA512:
-  metadata.gz: 703961acc8ac3cea16270b0905599edb4a2cd954b1ce7de57cd0291bf24e74a2eec5df6d0241bc67c34f8d8b4a92792a13159b38e907962ce64e31588854706e
-  data.tar.gz: 615928e9bea64f63fe1810c01592047cc70fc3ba118760b96e1991240002d7bbe72cec029ec5e841c9659c405d645972e7c0187bb7cb00e49f3cb3564df16291
+  metadata.gz: cd7ff25272e3ebf873603eb88db635772e0efc0642f6fde9addebb534a15c4ab9c4aa10271ebb4efed72be4c9dc0ee434e17d57fe2c5410d10353a1bf6c89ce1
+  data.tar.gz: '09bc159b740eb05dd2eb335308ab3f5b4b4842beef822014ac2525f7e7354cac607246cd6095d040853f96fc7171cca4bf4def21df044e4de2615af1a0525be9'

data/.github/workflows/tests.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [ '2.5', '2.6', '2.7' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
     name: Ruby ${{ matrix.ruby }} Tests
     steps:
       - name: Checkout code

data/.rubocop.yml

@@ -16,5 +16,8 @@ Naming/FileName:
 Lint/RescueException:
   Enabled: false
 
+Lint/MissingSuper:
+  Enabled: false
+
 Metrics/ParameterLists:
   Enabled: false

data/Gemfile.lock

@@ -1,43 +1,43 @@
 PATH
   remote: .
   specs:
-    heap-profiler (0.2.1)
+    heap-profiler (0.3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.1)
-    benchmark-ips (2.8.2)
+    benchmark-ips (2.8.4)
     byebug (11.1.3)
-    minitest (5.14.1)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    minitest (5.14.3)
+    parallel (1.20.1)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
     rainbow (3.0.0)
-    rake (13.0.1)
+    rake (13.0.3)
     rake-compiler (1.1.1)
       rake
-    regexp_parser (1.7.1)
+    regexp_parser (2.0.3)
     rexml (3.2.4)
-    rubocop (0.86.0)
+    rubocop (1.8.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.2.0)
-      parser (>= 2.7.0.1)
-    rubocop-shopify (1.0.4)
-      rubocop (>= 0.85, < 0.87)
-    ruby-progressbar (1.10.1)
-    stackprof (0.2.15)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.0)
+      parser (>= 2.7.1.5)
+    rubocop-shopify (1.0.7)
+      rubocop (~> 1.4)
+    ruby-progressbar (1.11.0)
+    stackprof (0.2.16)
+    unicode-display_width (2.0.0)
 
 PLATFORMS
-  ruby
+  x86_64-darwin-19
 
 DEPENDENCIES
   benchmark-ips
@@ -50,4 +50,4 @@ DEPENDENCIES
   stackprof
 
 BUNDLED WITH
-   2.1.4
+   2.2.5

data/ext/heap_profiler/extconf.rb

@@ -2,6 +2,8 @@
 
 require "mkmf"
 
+have_func("rb_enc_interned_str", "ruby.h")
+
 $CXXFLAGS += ' -O3 -std=c++1z -Wno-register '
 
 create_makefile 'heap_profiler/heap_profiler'

data/ext/heap_profiler/heap_profiler.cpp

@@ -1,4 +1,5 @@
 #include "ruby.h"
+#include "ruby/encoding.h"
 #include "simdjson.h"
 #include <fstream>
 
@@ -6,7 +7,7 @@ using namespace simdjson;
 
 static VALUE rb_eHeapProfilerError, sym_type, sym_class, sym_address, sym_value,
              sym_memsize, sym_imemo_type, sym_struct, sym_file, sym_line, sym_shared,
-             sym_references;
+             sym_references, id_uminus;
 
 typedef struct {
     dom::parser *parser;
@@ -83,6 +84,24 @@ static inline int64_t parse_dom_address(dom::element element) {
     return parse_address(address);
 }
 
+static inline VALUE make_symbol(std::string_view string) {
+    return ID2SYM(rb_intern2(string.data(), string.size()));
+}
+
+static inline VALUE make_string(std::string_view string) {
+    return rb_utf8_str_new(string.data(), string.size());
+}
+
+# ifdef HAVE_RB_ENC_INTERNED_STR
+    static inline VALUE dedup_string(std::string_view string) {
+        return rb_enc_interned_str(string.data(), string.size(), rb_utf8_encoding());
+    }
+# else
+    static inline VALUE dedup_string(std::string_view string) {
+        return rb_funcall(make_string(string), id_uminus, 0);
+    }
+# endif
+
 static VALUE rb_heap_build_index(VALUE self, VALUE path, VALUE batch_size) {
     Check_Type(path, T_STRING);
     Check_Type(batch_size, T_FIXNUM);
@@ -107,14 +126,31 @@ static VALUE rb_heap_build_index(VALUE self, VALUE path, VALUE batch_size) {
                 std::string_view value;
                 if (!object["value"].get(value)) {
                     VALUE address = INT2FIX(parse_dom_address(object["address"]));
-                    VALUE string = rb_utf8_str_new(value.data(), value.size());
+                    VALUE string = make_string(value);
                     rb_hash_aset(string_index, address, string);
                 }
             } else if (type == "CLASS" || type == "MODULE") {
+                VALUE address = INT2FIX(parse_dom_address(object["address"]));
+                VALUE class_name = Qfalse;
+
                 std::string_view name;
                 if (!object["name"].get(name)) {
-                    VALUE address = INT2FIX(parse_dom_address(object["address"]));
-                    VALUE class_name = rb_utf8_str_new(name.data(), name.size());
+                    class_name = dedup_string(name);
+                } else {
+                    std::string_view file;
+                    uint64_t line;
+
+                    if (!object["file"].get(file) && !object["line"].get(line)) {
+                        std::string buffer = "<Class ";
+                        buffer += file;
+                        buffer += ":";
+                        buffer += std::to_string(line);
+                        buffer += ">";
+                        class_name = dedup_string(buffer);
+                    }
+                }
+
+                if (RTEST(class_name)) {
                     rb_hash_aset(class_index, address, class_name);
                 }
             }
@@ -140,7 +176,7 @@ static VALUE make_ruby_object(dom::object object)
 
     std::string_view type;
     if (!object["type"].get(type)) {
-        rb_hash_aset(hash, sym_type, ID2SYM(rb_intern2(type.data(), type.size())));
+        rb_hash_aset(hash, sym_type, make_symbol(type));
     }
 
     std::string_view address;
@@ -164,17 +200,17 @@ static VALUE make_ruby_object(dom::object object)
     if (type == "IMEMO") {
         std::string_view imemo_type;
         if (!object["imemo_type"].get(imemo_type)) {
-            rb_hash_aset(hash, sym_imemo_type, ID2SYM(rb_intern2(imemo_type.data(), imemo_type.size())));
+            rb_hash_aset(hash, sym_imemo_type, make_symbol(imemo_type));
         }
     } else if (type == "DATA") {
         std::string_view _struct;
         if (!object["struct"].get(_struct)) {
-            rb_hash_aset(hash, sym_struct, ID2SYM(rb_intern2(_struct.data(), _struct.size())));
+            rb_hash_aset(hash, sym_struct, make_symbol(_struct));
         }
     } else if (type == "STRING") {
         std::string_view value;
         if (!object["value"].get(value)) {
-            rb_hash_aset(hash, sym_value, rb_utf8_str_new(value.data(), value.size()));
+            rb_hash_aset(hash, sym_value, make_string(value));
         }
 
         bool shared;
@@ -196,7 +232,7 @@ static VALUE make_ruby_object(dom::object object)
 
     std::string_view file;
     if (!object["file"].get(file)) {
-        rb_hash_aset(hash, sym_file, rb_utf8_str_new(file.data(), file.size()));
+        rb_hash_aset(hash, sym_file, dedup_string(file));
     }
 
     uint64_t line;
@@ -261,6 +297,7 @@ extern "C" {
         sym_line = ID2SYM(rb_intern("line"));
         sym_shared = ID2SYM(rb_intern("shared"));
         sym_references = ID2SYM(rb_intern("references"));
+        id_uminus = rb_intern("-@");
 
         VALUE rb_mHeapProfiler = rb_const_get(rb_cObject, rb_intern("HeapProfiler"));
 

data/lib/heap_profiler/dump.rb

@@ -61,7 +61,7 @@ module HeapProfiler
       end
     end
 
-    def each_object(since: 0, &block)
+    def each_object(since: nil, &block)
       Parser.load_many(path, since: since, batch_size: 10_000_000, &block)
     end
 

data/lib/heap_profiler/index.rb

@@ -34,7 +34,7 @@ module HeapProfiler
     }.freeze
 
     IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }
-    DATA_TYPES = Hash.new { |h, k| h[k] = "<#{(k || 'unknown')}> (DATA)" }
+    DATA_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (DATA)" }
 
     def guess_class(object)
       type = object[:type]
@@ -44,14 +44,15 @@ module HeapProfiler
 
       return IMEMO_TYPES[object[:imemo_type]] if type == :IMEMO
 
-      class_address = object[:class]
-      return unless class_address
+      if (class_address = object[:class])
+        @classes.fetch(class_address) do
+          return DATA_TYPES[object[:struct]] if type == :DATA
 
-      @classes.fetch(class_address) do
-        return DATA_TYPES[object[:struct]] if type == :DATA
-
-        $stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
-        nil
+          $stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
+          nil
+        end
+      elsif type == :DATA
+        DATA_TYPES[object[:struct]]
       end
     end
 

data/lib/heap_profiler/parser.rb

@@ -2,18 +2,28 @@
 
 module HeapProfiler
   module Parser
+    CLASS_DEFAULT_PROC = ->(_hash, key) { "<Class#0x#{key.to_s(16)}>" }
+
     class Ruby
       def build_index(path)
         require 'json'
         classes_index = {}
+        classes_index.default_proc = CLASS_DEFAULT_PROC
         strings_index = {}
 
         File.open(path).each_line do |line|
           object = JSON.parse(line, symbolize_names: true)
           case object[:type]
           when 'MODULE', 'CLASS'
-            if (name = object[:name])
-              classes_index[parse_address(object[:address])] = name
+            address = parse_address(object[:address])
+
+            name = object[:name]
+            name ||= if object[:file] && object[:line]
+              "<Class #{object[:file]}:#{object[:line]}>"
+            end
+
+            if name
+              classes_index[address] = name
             end
           when 'STRING'
             next if object[:shared]
@@ -35,7 +45,9 @@ module HeapProfiler
       DEFAULT_BATCH_SIZE = 10_000_000 # 10MB
 
       def build_index(path, batch_size: DEFAULT_BATCH_SIZE)
-        _build_index(path, batch_size)
+        indexes = _build_index(path, batch_size)
+        indexes.first.default_proc = CLASS_DEFAULT_PROC
+        indexes
       end
 
       def load_many(path, since: nil, batch_size: DEFAULT_BATCH_SIZE, &block)

data/lib/heap_profiler/reporter.rb

@@ -9,8 +9,6 @@ module HeapProfiler
     # So we name them at the start of the profile to avoid that.
     #
     # See: https://github.com/ruby/ruby/pull/3349
-    #
-    # TODO: Could we actually do the dump ourselves? objspace is a extension already.
     if RUBY_VERSION < '2.8'
       def name_anonymous_modules!
         ObjectSpace.each_object(Module) do |mod|
@@ -56,10 +54,14 @@ module HeapProfiler
     def stop
       HeapProfiler.name_anonymous_modules!
       ObjectSpace.trace_object_allocations_stop if @enable_tracing
+
+      # we can't use partial dump for allocated.heap, because we need old generations
+      # as well to build the classes and strings indexes.
       dump_heap(@allocated_heap)
+
       GC.enable
       4.times { GC.start }
-      dump_heap(@retained_heap)
+      dump_heap(@retained_heap, partial: true)
       @allocated_heap.close
       @retained_heap.close
       write_info("generation", @generation.to_s)
@@ -84,17 +86,24 @@ module HeapProfiler
       File.write(File.join(@dir_path, "#{key}.info"), value)
     end
 
-    # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
-    # because of this even en empty block of code will report a handful of allocations.
-    # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
-    # property.
-    class_eval <<~RUBY, '__hprof', __LINE__
-      # frozen_string_literal: true
-      def dump_heap(file)
-        ObjectSpace.dump_all(output: file)
+    if RUBY_VERSION >= '3.0'
+      def dump_heap(file, partial: false)
+        ObjectSpace.dump_all(output: file, since: partial ? @generation : nil)
         file.close
       end
-    RUBY
+    else
+      # ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
+      # because of this even en empty block of code will report a handful of allocations.
+      # To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
+      # property.
+      class_eval <<~RUBY, '__hprof', __LINE__
+        # frozen_string_literal: true
+        def dump_heap(file, partial: false)
+          ObjectSpace.dump_all(output: file)
+          file.close
+        end
+      RUBY
+    end
 
     def open_heap(name)
       File.open(File.join(@dir_path, "#{name}.heap"), 'w+')

data/lib/heap_profiler/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module HeapProfiler
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heap-profiler
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies: []
 description: Make several heap dumps and summarize allocated, retained memory
 email: