heapy 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56f148e16da6c3dd3e07b1db360eb39e3ff9d076
-  data.tar.gz: a00043f1abb083c6902f5ad9a113c877a2de1857
+  metadata.gz: ababd2495856e5682c0f7f136ee0fe112304bc1a
+  data.tar.gz: db47b39aa2d4207dc27a1753c1444830aba758bb
 SHA512:
-  metadata.gz: 7afc522e6d2fe8f666ebf00bfa8f002be1bc38c98891fdfee88f5cb407d45ec3701fa09f4c2adcf1f6a98466f48cd0332f9479888e927bc2c53457560ab1300f
-  data.tar.gz: ea9e2a37f6e00973b820f526b894cfb4637e15f20135a5aceed2b87ad71634290b004c13c67542f050f6473145a180bf2e2d17207d46e9a1faf98c045f5826b1
+  metadata.gz: 5695269c2619549c931188225fd6b52f0d2acd343f92aba069a8e3cd937d0c97d2b6b15f2585bd96ae78c905708b8612a0ff51bcbfb4ce746f6bd6df423c62f4
+  data.tar.gz: b70fbde861babd816b71ee8da6c8ecafb06192740dc4dcbdbb19ad930578925d62ffbdf295566f9e2d938366e385281303a4b1c16ad86c9486ff6f4bb8131d13

data/README.md

@@ -27,16 +27,35 @@ Step 2) Once you've got the heap dump, you can analyze it using this CLI:
 ```
 $ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump
 
-Generation:  0 object count: 209191
-Generation: 14 object count: 407
-Generation: 15 object count: 638
-Generation: 16 object count: 748
-Generation: 17 object count: 1023
-Generation: 18 object count: 805
+Generation: nil object count: 209191
+Generation:  14 object count: 407
+Generation:  15 object count: 638
+Generation:  16 object count: 748
+Generation:  17 object count: 1023
+Generation:  18 object count: 805
 # ...
 ```
 
-Generally early generations will have a high object count as an app is initialized. Over time however, the object count should stabalize. If however you see it spike up, you can drill down into a specific generation. In the previous example, the 17'th generation looks strangely large, you can drill into it:
+NOTE: The reason you may be getting a "nil" generation is these objects were loaded into memory before your code began tracking the allocations. To ensure all allocations are tracked you can execute your ruby script this trick. First create a file `trace.rb` that only starts allocation tracing:
+
+```
+# trace.rb
+require 'objspace'
+
+ObjectSpace.trace_object_allocations_start
+```
+
+Now make sure this command is loaded before you run your script, you can use Ruby's `-I` to specify a load path and `-r` to specify a library to require, in this case our trace file
+
+```
+$ ruby -I ./ -r trace script_name.rb
+```
+
+If the last line of your file is invalid JSON, make sure that you are closing the file after writing the ruby heap dump to it.
+
+## Digging into a Generation
+
+You can drill down into a specific generation. In the previous example, the 17'th generation looks strangely large, you can drill into it:
 
 
 ```
@@ -54,6 +73,12 @@ $ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump 17
          92200  /app/vendor/bundle/ruby/2.2.0/gems/activesupport-4.2.3/lib/active_support/core_ext/numeric/conversions.rb:131
 ```
 
+If you want to read all generations you can use the "all" directive
+
+```
+$ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump all
+```
+
 You can also use T-Lo's online JS based [Heap Analyzer](http://tenderlove.github.io/heap-analyzer/) for visualizations.
 
 ## Development

data/lib/heapy.rb

@@ -19,12 +19,12 @@ $ heapy read <file|command> <number>
 When run with only a file, it will output the generation and count pairs:
 
   $ heapy read tmp/2015-09-30-heap.dump
-    Generation:  0 object count: 209191
-    Generation: 14 object count: 407
-    Generation: 15 object count: 638
-    Generation: 16 object count: 748
-    Generation: 17 object count: 1023
-    Generation: 18 object count: 805
+    Generation: nil object count: 209191
+    Generation:  14 object count: 407
+    Generation:  15 object count: 638
+    Generation:  16 object count: 748
+    Generation:  17 object count: 1023
+    Generation:  18 object count: 805
 
 When run with a file and a number it will output detailed information for that
 generation:
@@ -69,38 +69,61 @@ HALP
       @filename = filename
     end
 
-    def drill_down(generation)
-      puts ""
-      puts "Analyzing Heap (Generation: #{generation})"
-      puts "-------------------------------"
-      puts ""
-
-      generation = Integer(generation)
-
-      #
-      memsize_hash = Hash.new { |h, k| h[k] = 0 }
-      count_hash   = Hash.new { |h, k| h[k] = 0 }
+    def read
       File.open(@filename) do |f|
         f.each_line do |line|
           begin
             parsed = JSON.parse(line)
-            if parsed["generation"] == generation
-              key = "#{ parsed["file"] }:#{ parsed["line"] }"
-              memsize_hash[key] += parsed["memsize"]
-              count_hash[key]   += 1
-            end
+            yield parsed
           rescue JSON::ParserError
             puts "Could not parse #{line}"
           end
         end
       end
+    end
+
+    def drill_down(generation_to_inspect)
+      puts ""
+      puts "Analyzing Heap (Generation: #{generation_to_inspect})"
+      puts "-------------------------------"
+      puts ""
+
+      generation_to_inspect = Integer(generation_to_inspect) unless generation_to_inspect == "all"
+
+      #
+      memsize_hash    = Hash.new { |h, k| h[k] = 0  }
+      count_hash      = Hash.new { |h, k| h[k] = 0  }
+      string_count    = Hash.new { |h, k| h[k] = Hash.new { |h, k| h[k] = 0  } }
+
+      reference_hash  = Hash.new { |h, k| h[k] = 0  }
+
+      read do |parsed|
+        generation = parsed["generation"] || 0
+        if generation_to_inspect == "all".freeze || generation == generation_to_inspect
+          next unless parsed["file"]
+
+          key = "#{ parsed["file"] }:#{ parsed["line"] }"
+          memsize_hash[key] += parsed["memsize"] || 0
+          count_hash[key]   += 1
+
+          if parsed["type"] == "STRING".freeze
+            string_count[parsed["value"]][key] += 1 if parsed["value"]
+          end
+
+          if parsed["references"]
+            reference_hash[key] += parsed["references"].length
+          end
+        end
+      end
+
+      raise "not a valid Generation: #{generation_to_inspect.inspect}" if memsize_hash.empty?
 
       total_memsize = memsize_hash.inject(0){|count, (k, v)| count += v}
 
       # /Users/richardschneeman/Documents/projects/codetriage/app/views/layouts/application.html.slim:1"=>[{"address"=>"0x7f8a4fbf2328", "type"=>"STRING", "class"=>"0x7f8a4d5dec68", "bytesize"=>223051, "capacity"=>376832, "encoding"=>"UTF-8", "file"=>"/Users/richardschneeman/Documents/projects/codetriage/app/views/layouts/application.html.slim", "line"=>1, "method"=>"new", "generation"=>36, "memsize"=>377065, "flags"=>{"wb_protected"=>true, "old"=>true, "long_lived"=>true, "marked"=>true}}]}
       puts "allocated by memory (#{total_memsize}) (in bytes)"
       puts "=============================="
-      memsize_hash = memsize_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }
+      memsize_hash = memsize_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
       longest      = memsize_hash.first[1].to_s.length
       memsize_hash.each do |file_line, memsize|
         puts "  #{memsize.to_s.rjust(longest)}  #{file_line}"
@@ -110,37 +133,66 @@ HALP
 
       puts ""
       puts "object count (#{total_count})"
-      puts "============"
-      count_hash = count_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }
+      puts "=============================="
+      count_hash = count_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
       longest      = count_hash.first[1].to_s.length
       count_hash.each do |file_line, memsize|
         puts "  #{memsize.to_s.rjust(longest)}  #{file_line}"
       end
+
+      puts ""
+      puts "High Ref Counts"
+      puts "=============================="
+      puts ""
+
+      reference_hash = reference_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+      longest      = count_hash.first[1].to_s.length
+
+      reference_hash.each do |file_line, count|
+        puts "  #{count.to_s.rjust(longest)}  #{file_line}"
+      end
+
+      puts ""
+      puts "Duplicate strings"
+      puts "=============================="
+      puts ""
+      value_count = {}
+
+      string_count.each do |string, location_count_hash|
+        value_count[string] = location_count_hash.values.inject(&:+)
+      end
+
+      value_count = value_count.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+      longest     = value_count.first[1].to_s.length
+
+      value_count.each do |string, c1|
+
+        puts " #{c1.to_s.rjust(longest)}  #{string.inspect}"
+        string_count[string].sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.each do |file_line, c2|
+         puts " #{c2.to_s.rjust(longest)}  #{file_line}"
+       end
+       puts ""
+      end
+
     end
 
     def analyze
       puts ""
       puts "Analyzing Heap"
       puts "=============="
+      default_key = "nil".freeze
 
       # generation number is key, value is count
       data = Hash.new {|h, k| h[k] = 0 }
 
-      File.open(@filename) do |f|
-        f.each_line do |line|
-          begin
-            line = line.chomp
-            json = JSON.parse(line)
-            data[json["generation"]||0] += 1
-          rescue JSON::ParserError
-            puts "Could not parse #{line}"
-          end
-        end
+      read do |parsed|
+        data[parsed["generation"] || 0] += 1
       end
 
-      data = data.sort
-      max_length = data.last[0].to_s.length
+      data = data.sort {|(k1,v1), (k2,v2)| k1 <=> k2 }
+      max_length = [data.last[0].to_s.length, default_key.length].max
       data.each do |generation, count|
+        generation = default_key if generation == 0
         puts "Generation: #{ generation.to_s.rjust(max_length) } object count: #{ count }"
       end
     end

data/lib/heapy/version.rb

@@ -1,3 +1,3 @@
 module Heapy
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/scratch.rb

@@ -0,0 +1,16 @@
+require 'objspace'
+
+ObjectSpace.trace_object_allocations_start
+
+array = []
+10_000.times do |x|
+  a = "#{x}_foo"
+  array << a
+end
+
+# GC.start
+
+file_name = "/tmp/#{Time.now.to_f}-heap.dump"
+ObjectSpace.dump_all(output: File.open(file_name, 'w'))
+
+puts "bin/heapy read #{file_name}"

data/trace.rb

@@ -0,0 +1,3 @@
+require 'objspace'
+
+ObjectSpace.trace_object_allocations_start

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heapy
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - schneems
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-15 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -74,6 +74,8 @@ files:
 - heapy.gemspec
 - lib/heapy.rb
 - lib/heapy/version.rb
+- scratch.rb
+- trace.rb
 homepage: https://github.com/schneems/heapy
 licenses:
 - MIT
@@ -94,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.0
 signing_key: 
 specification_version: 4
 summary: Inspects Ruby heap dumps