heapy 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 642cdc83f817b16ef0fc8e35902a37bc197edd8edb0f14a1bab68920d2bf4d89
-  data.tar.gz: f98f9629ace983a246ff4f2cadf5749e41297ba1cdf2da76d0712c4f4a36fc80
+  metadata.gz: a84bd36786d08056d5e172044cbbe30547ca5fc911aeee1a6a46905daef45c7d
+  data.tar.gz: b87cd42efd90097653dbe311955e66c45b961828e8c4538a2604d23f66bc8ba1
 SHA512:
-  metadata.gz: 219b46ee0b8162fe40faac68d010f6c9867a2242920fd62056a7eac2a4f5739f2c5221e9067aec01328da931d2283feb5541eb571e4f401f8fa8a75f7c30bac8
-  data.tar.gz: 48e4813a71c561866033544b7ae6f0fcfea4bfda45096dbe3977ed9a3244ccb93741533cb52d6648109f88dd9222486a763ee75deb3e6ae242423bb82ea3aa2b
+  metadata.gz: bc703b95887ec3343cf21fc75e68ade763a763c357b7ef4e3e6dcc94d18751559241a587ebb9160321dfb2a975e51707ac5a030ba450dd76b8cf3ba31f385660
+  data.tar.gz: b13485e91fdbe6922e7777c7a8acfd48d3be5433891e2e994d38fa6c6c59881ae6085e532e9b9a04e82d6a80a005ef73afbe684ee2efe27fcbc85a402aeb5a17

data/.github/workflows/check_changelog.yml

@@ -0,0 +1,13 @@
+name: Check Changelog
+
+on:
+ pull_request:
+  types: [opened, reopened, edited, synchronize]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check that CHANGELOG is touched
+      run: |
+        cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 tmp
 .DS_Store
+scratch.rb

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## HEAD
+
+## 0.2.0
+
+- Heapy::Alive is removed (https://github.com/schneems/heapy/pull/27)
+- New command `heapy diff` (https://github.com/schneems/heapy/pull/26)
+- The read command now takes a --lines flag that can be used to limit output when showing generational data
+
 ## 0.1.4 - 2018-07-25
 
 - Bundler is no longer required so heapy can now be used via a simple

data/README.md

@@ -21,6 +21,34 @@ Or install it yourself as:
 
 ## Usage
 
+### Diff 2 heap dumps
+
+Run with two inputs to output the values of today.dump that are not present in yesterday.dump
+
+```
+$ heapy diff tmp/yesterday.dump tmp/today_morning.dump
+Allocated STRING 9991 objects of size 399640/491264 (in bytes) at: scratch.rb:24
+```
+
+Run with three inputs to show the diff between the first two, but only if the objects are still retained in the third
+
+```
+$ heapy diff tmp/yesterday.dump tmp/today_morning.dump tmp/today_afternoon.dump
+Retained STRING 9991 objects of size 399640/491264 (in bytes) at: scratch.rb:24
+# ...
+```
+
+Pass in the name of an output file and the objects present in today.dump that aren't in yesterday.dump will be written to that file
+
+```
+$ heapy diff tmp/yesterday.dump tmp/today.dump --output_diff=output.json
+Allocated STRING 9991 objects of size 399640/491264 (in bytes) at: scratch.rb:24
+# ...
+Writing heap dump diff to output.json
+```
+
+### Read a Heap Dump
+
 Step 1) Generate a heap dump. You could [do this manually](http://samsaffron.com/archive/2015/03/31/debugging-memory-leaks-in-ruby). Or you can use a tool like [derailed_benchmarks](https://github.com/schneems/derailed_benchmarks)
 
 Step 2) Once you've got the heap dump, you can analyze it using this CLI:
@@ -54,11 +82,10 @@ $ ruby -I ./ -r trace script_name.rb
 
 If the last line of your file is invalid JSON, make sure that you are closing the file after writing the ruby heap dump to it.
 
-## Digging into a Generation
+### Digging into a Generation
 
 You can drill down into a specific generation. In the previous example, the 17'th generation looks strangely large, you can drill into it:
 
-
 ```
 $ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump 17
     Analyzing Heap (Generation: 17)
@@ -74,6 +101,14 @@ $ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump 17
          92200  /app/vendor/bundle/ruby/2.2.0/gems/activesupport-4.2.3/lib/active_support/core_ext/numeric/conversions.rb:131
 ```
 
+You can limit the output by passing in a `--lines` value:
+
+```
+$ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump 17 --lines=6
+```
+
+> Note: Default lines value is 50
+
 ### Reviewing all generations
 
 If you want to read all generations you can use the "all" directive
@@ -82,7 +117,7 @@ If you want to read all generations you can use the "all" directive
 $ heapy read tmp/2015-10-01T10:18:59-05:00-heap.dump all
 ```
 
-You can also use T-Lo's online JS based [Heap Analyzer](http://tenderlove.github.io/heap-analyzer/) for visualizations.
+You can also use T-Lo's online JS based [Heap Analyzer](http://tenderlove.github.io/heap-analyzer/) for visualizations. Another tool is [HARB](https://github.com/csfrancis/harb)
 
 ## Development
 
@@ -94,7 +129,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/schneems/heapy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/bin/heapy

@@ -3,5 +3,4 @@
 require "bundler/setup" if defined?(Bundler)
 require "heapy"
 
-
-Heapy::CLI.new(ARGV).run
+Heapy::CLI.start(ARGV)

data/heapy.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "bundler", "> 1"
+  spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec"
-  
-  spec.required_ruby_version = '~> 2.3'
 end

data/lib/heapy.rb

@@ -1,70 +1,105 @@
 require 'json'
+require 'thor'
 
 require "heapy/version"
 
 module Heapy
-
-  class CLI
-    def initialize(argv)
-      @cmd    = argv.shift
-      @file   = argv.shift
-      @number = argv.shift
-      @args   = argv
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
     end
 
-    def help
-      puts <<-HALP
-$ heapy read <file|command> <number>
-
-When run with only a file, it will output the generation and count pairs:
-
-  $ heapy read tmp/2015-09-30-heap.dump
-    Generation: nil object count: 209191
-    Generation:  14 object count: 407
-    Generation:  15 object count: 638
-    Generation:  16 object count: 748
-    Generation:  17 object count: 1023
-    Generation:  18 object count: 805
-
-When run with a file and a number it will output detailed information for that
-generation:
-
-  $ heapy read tmp/2015-09-30-heap.dump 17
-
-    Analyzing Heap (Generation: 17)
-    -------------------------------
-
-allocated by memory (44061517) (in bytes)
-==============================
-  39908512  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/timeout.rb:79
-   1284993  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/openssl/buffering.rb:182
-    201068  /app/vendor/bundle/ruby/2.2.0/gems/json-1.8.3/lib/json/common.rb:223
-    189272  /app/vendor/bundle/ruby/2.2.0/gems/newrelic_rpm-3.13.2.302/lib/new_relic/agent/stats_engine/stats_hash.rb:39
-    172531  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/net/http/header.rb:172
-     92200  /app/vendor/bundle/ruby/2.2.0/gems/activesupport-4.2.3/lib/active_support/core_ext/numeric/conversions.rb:131
-HALP
-    end
+    desc "read <file> <generation> --lines <number_of_lines>", "Read heap dump file"
+    long_desc <<-DESC
+      When run with only a file input, it will output the generation and count pairs:
+
+        $ heapy read tmp/2015-09-30-heap.dump\x5
+          Generation: nil object count: 209191\x5
+          Generation:  14 object count: 407\x5
+          Generation:  15 object count: 638\x5
+          Generation:  16 object count: 748\x5
+          Generation:  17 object count: 1023\x5
+          Generation:  18 object count: 805\x5
+
+      When run with a file and a number it will output detailed information for that\x5
+      generation:\x5
 
-    def run
-
-      case @cmd
-      when "--help"
-        help
-      when nil
-        help
-      when "read"
-        if @number
-          Analyzer.new(@file).drill_down(@number)
-        else
-          Analyzer.new(@file).analyze
-        end
+        $ heapy read tmp/2015-09-30-heap.dump 17\x5
+
+          Analyzing Heap (Generation: 17)\x5
+          -------------------------------\x5
+
+      allocated by memory (44061517) (in bytes)\x5
+      ==============================\x5
+        39908512  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/timeout.rb:79\x5
+         1284993  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/openssl/buffering.rb:182\x5
+          201068  /app/vendor/bundle/ruby/2.2.0/gems/json-1.8.3/lib/json/common.rb:223\x5
+          189272  /app/vendor/bundle/ruby/2.2.0/gems/newrelic_rpm-3.13.2.302/lib/new_relic/agent/stats_engine/stats_hash.rb:39\x5
+          172531  /app/vendor/ruby-2.2.3/lib/ruby/2.2.0/net/http/header.rb:172\x5
+           92200  /app/vendor/bundle/ruby/2.2.0/gems/activesupport-4.2.3/lib/active_support/core_ext/numeric/conversions.rb:131\x5
+    DESC
+    option :lines, required: false, :type => :numeric
+    def read(file_name, generation = nil)
+      if generation
+        Analyzer.new(file_name).drill_down(generation, options[:lines] || 50)
       else
-        help
+        Analyzer.new(file_name).analyze
       end
     end
+
+    long_desc <<-DESC
+      Run with two inputs to output the values of today.dump that are not present in yesterday.dump
+
+        $ heapy diff tmp/yesterday.dump tmp/today.dump\x5
+
+      Run with three inputs to show the diff between the first two, but only if the objects are still retained in the third
+
+        $ heapy diff tmp/yesterday.dump tmp/today_morning.dump tmp/today_afternoon.dump\x5
+
+      Pass in the name of an output file and the objects present in today.dump that aren't in yesterday.dump will be written to that file
+
+        $ heapy diff tmp/yesterday.dump tmp/today.dump --output_diff=output.json\x5
+
+    DESC
+    desc "diff <before_file> <after_file> <retained_file (optional)> --output_diff=output.json", "Diffs 2 heap dumps"
+    option :output_diff, required: false, :type => :string
+    def diff(before, after, retained = nil)
+      Diff.new(before: before, after: after, retained: retained, output_diff: options[:output_diff] || nil).call
+    end
+
+    map %w[--version -v] => :version
+    desc "version", "Show heapy version"
+    def version
+      puts Heapy::VERSION
+    end
+
+    desc "wat", "Outputs instructions on how to make a manual heap dump"
+    def wat
+      puts <<-HELP
+
+To get a heap dump do this:
+
+    require 'objspace'
+    ObjectSpace.trace_object_allocations_start
+
+    # Your code here
+
+    p ObjectSpace.dump_all
+
+    # => #<File:/path/to/output/heap/dump/here.json>
+
+This will print the file name of your heap dump.
+
+If you prefer you can manually pass in an IO object to `ObjectSpace.dump_all`
+
+    io = File.open("/tmp/my_dump.json", "w+")
+    ObjectSpace.dump_all(output: io);
+    io.close
+
+HELP
+    end
   end
 end
 
 require 'heapy/analyzer'
-require 'heapy/alive'
-
+require 'heapy/diff'

data/lib/heapy/analyzer.rb

@@ -1,4 +1,14 @@
 module Heapy
+
+  # Used for inspecting contents of a heap dump
+  #
+  # To glance all contents at a glance run:
+  #
+  #   Analyzer.new(file_name).analyze
+  #
+  # To inspect contents of a specific generation run:
+  #
+  #   Analyzer.new(file_name).drill_down(generation, Float::INFINITY)
   class Analyzer
     def initialize(filename)
       @filename = filename
@@ -17,7 +27,7 @@ module Heapy
       end
     end
 
-    def drill_down(generation_to_inspect)
+    def drill_down(generation_to_inspect, max_items_to_display)
       puts ""
       puts "Analyzing Heap (Generation: #{generation_to_inspect})"
       puts "-------------------------------"
@@ -25,7 +35,6 @@ module Heapy
 
       generation_to_inspect = Integer(generation_to_inspect) unless generation_to_inspect == "all"
 
-      #
       memsize_hash    = Hash.new { |h, k| h[k] = 0  }
       count_hash      = Hash.new { |h, k| h[k] = 0  }
       string_count    = Hash.new { |h, k| h[k] = Hash.new { |h, k| h[k] = 0  } }
@@ -58,7 +67,7 @@ module Heapy
       # /Users/richardschneeman/Documents/projects/codetriage/app/views/layouts/application.html.slim:1"=>[{"address"=>"0x7f8a4fbf2328", "type"=>"STRING", "class"=>"0x7f8a4d5dec68", "bytesize"=>223051, "capacity"=>376832, "encoding"=>"UTF-8", "file"=>"/Users/richardschneeman/Documents/projects/codetriage/app/views/layouts/application.html.slim", "line"=>1, "method"=>"new", "generation"=>36, "memsize"=>377065, "flags"=>{"wb_protected"=>true, "old"=>true, "long_lived"=>true, "marked"=>true}}]}
       puts "allocated by memory (#{total_memsize}) (in bytes)"
       puts "=============================="
-      memsize_hash = memsize_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+      memsize_hash = memsize_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(max_items_to_display)
       longest      = memsize_hash.first[1].to_s.length
       memsize_hash.each do |file_line, memsize|
         puts "  #{memsize.to_s.rjust(longest)}  #{file_line}"
@@ -69,7 +78,7 @@ module Heapy
       puts ""
       puts "object count (#{total_count})"
       puts "=============================="
-      count_hash = count_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+      count_hash = count_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(max_items_to_display)
       longest      = count_hash.first[1].to_s.length
       count_hash.each do |file_line, memsize|
         puts "  #{memsize.to_s.rjust(longest)}  #{file_line}"
@@ -80,7 +89,7 @@ module Heapy
       puts "=============================="
       puts ""
 
-      reference_hash = reference_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+      reference_hash = reference_hash.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(max_items_to_display)
       longest      = count_hash.first[1].to_s.length
 
       reference_hash.each do |file_line, count|
@@ -99,7 +108,7 @@ module Heapy
           value_count[string] = location_count_hash.values.inject(&:+)
         end
 
-        value_count = value_count.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50)
+        value_count = value_count.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(max_items_to_display)
         longest     = value_count.first[1].to_s.length
 
         value_count.each do |string, c1|

data/lib/heapy/diff.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'json'
+module Heapy
+  # Diff 2 dumps example:
+  #
+  #   Heapy::Diff.new(before: 'my_dump_1.json', after: 'my_dump_2.json').call
+  #
+  # This will find objects that are present in my_dump_2 that are not present in my_dump_1
+  # this means they were allocated sometime between the two heap dumps.
+  #
+  # Diff 3 dumps example:
+  #
+  #   Heapy::Diff.new(before: 'my_dump_1.json', after: 'my_dump_2.json', retained: 'my_dump_3.json').call
+  #
+  # This will find objects that are present in my_dump_2 that are not present in my_dump_1
+  # but only if the objects are still present at the time that my_dump_3 was taken. This does
+  # not guarantee that they're retained forever, but were still present at the time the last
+  # dump was taken.
+  #
+  # You can output the diff of heap dumps by passing in a filename as `output_diff` for example
+  #
+  #   Heapy::Diff.new(before: 'my_dump_1.json', after: 'my_dump_2.json', outpu_diff: 'out.json').call
+  class Diff
+    attr_reader :diff
+
+    def initialize(before:, after:, retained: nil, io: STDOUT, output_diff: nil)
+      @before_file = before
+      @after_file = after
+      @retained_file = retained
+      @output_diff_file = output_diff ? File.open(output_diff, "w+") : nil
+      @io = io
+      @diff = Hash.new { |hash, k|
+        hash[k] = {}
+        hash[k]["count"] = 0
+        hash[k]["memsize"] = 0
+        hash[k]
+      }
+
+      @before_address_hash = {}
+      @retained_address_hash = {}
+    end
+
+
+    def call
+      read(@before_file) { |parsed| @before_address_hash[parsed['address']] = true }
+      read(@retained_file) { |parsed| @retained_address_hash[parsed['address']] = true } if @retained_file
+
+      read(@after_file) do |parsed, original_line|
+        address = parsed['address']
+        next if previously_allocated?(address)
+        next if not_retained?(address)
+
+        @output_diff_file.puts original_line if @output_diff_file
+
+        hash = diff["#{parsed['type']},#{parsed['file']},#{parsed['line']}"]
+        hash["count"] += 1
+        hash["memsize"] += parsed["memsize"] || 0
+        hash["type"] ||= parsed["type"]
+        hash["file"] ||= parsed["file"]
+        hash["line"] ||= parsed["line"]
+      end
+
+      @output_diff_file.close if @output_diff_file
+      @before_address_hash.clear
+      @retained_address_hash.clear
+
+      total_memsize = diff.inject(0){|sum,(_,v)| sum + v["memsize"] }
+
+      diff.sort_by do |k,v|
+        v["count"]
+      end.reverse.each do |key, data|
+        @io.puts "#{@retained_file ? "Retained" : "Allocated"} #{data['type']} #{data['count']} objects of size #{data['memsize']}/#{total_memsize} (in bytes) at: #{data['file']}:#{data['line']}"
+      end
+
+      @io.puts "\nWriting heap dump diff to #{@output_diff_file.path}\n" if @output_diff_file
+    end
+
+    private def is_retained?(address)
+      return true if @retained_file.nil?
+      @retained_address_hash[address]
+    end
+
+    private def not_retained?(address)
+      !is_retained?(address)
+    end
+
+    private def previously_allocated?(address)
+      @before_address_hash[address]
+    end
+
+    private def read(filename)
+      File.open(filename) do |f|
+        f.each_line do |line|
+          begin
+            parsed = JSON.parse(line)
+            yield parsed, line
+          rescue JSON::ParserError
+            puts "Could not parse #{line}"
+          end
+        end
+      end
+    end
+  end
+end