heapviz 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2187fa754ebc6e334dabd3ba02614c7ba986ca5e118e94008a7afdbb47fa8daf
+  data.tar.gz: 4b72b34cbf9a0603ca3df40ce45b915a8c94c8b194bd52e0d655f51f3c2d5140
+SHA512:
+  metadata.gz: 37bdd98fad55a7025f7208fea532775c231f7a0caac2bbdf2058444becd03f7939a902bd1693c2b613e138527210c2bf866d260d1e969200ba9eb12f2ac911ab
+  data.tar.gz: 984326ab4d1cfd28f1d9cf3689c21b411706d4af2bbe168484bf7473729e1de1495dcf7245142954499557af4329b57f600aa93b65003c1a912917fc8aedda9a

data/.ruby-version

@@ -0,0 +1 @@
+3.1.0

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "minitest", "~> 5.0"

data/Gemfile.lock

@@ -0,0 +1,27 @@
+PATH
+  remote: .
+  specs:
+    heapviz (0.1.0)
+      chunky_png (~> 1.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    chunky_png (1.4.0)
+    minitest (5.15.0)
+    mocha (1.13.0)
+    rake (13.0.6)
+
+PLATFORMS
+  arm64-darwin-21
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  heapviz!
+  minitest (~> 5.0)
+  mocha (~> 1.13.0)
+  rake (~> 13.0)
+
+BUNDLED WITH
+   2.3.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Matt Valentine-House
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# Heapviz
+
+**Warning: This probably shouldn't be used for anything important. It
+requires patching and building a custom Ruby**
+
+Produces a png of a json dump of the Ruby objspace. Heap dump images
+are organised in rows and columns:
+
+* A heap page is a vertical column 4 pixels wide
+* A slot is a vertical chunk that is 4 pixels wide and
+  (4*slot_size/sizeof(RVALUE)) tall. so 40 byte slots are 4x4 pixels,
+  80 bytes slots are 4x8 pixels, 160 bytes are 4x16 pixels and so on.
+
+## Running
+
+This script requires a commit that recently landed on Ruby master to
+add a `slot_size` field to the output of `ObjectSpace.dump_all`. Use
+your Ruby version manager of choice to install the development release
+of Ruby and then write a script that dumps the heap to a file.
+
+```
+require 'objspace'
+ObjectSpace.dump_all(output: File.new("output.json", "w"))
+```
+
+Run it with Ruby
+
+```
+./ruby test.rb
+```
+
+Then run this script with your json output, and a filename of a png
+to write to (the png doesn't need to exist)
+
+```
+../heapviz/exe/heapviz output.json output.png
+open output.png
+```

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "heapviz"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/heapviz

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.prepend("#{__dir__}/../lib")
+require 'heapviz'
+
+if ARGV.length != 2
+  p "Usage: heapviz <path_to_heap_dump.json> <output.png>"
+  exit 1
+end
+
+heap = Heapviz::Heap.new
+
+File.open(ARGV[0]) do |f|
+  f.each_line do |line|
+    live_object = Heapviz::Slot.new(line)
+
+    if !live_object.gc_root?
+      page = heap.get_or_build_page_for(live_object)
+      page.fill_slot(live_object)
+    end
+  end
+end
+
+Heapviz::Renderer.new(ARGV[1], heap).render

data/lib/heapviz/config.rb

@@ -0,0 +1,13 @@
+require 'fiddle'
+
+module Heapviz
+  module Config
+    SIZEOF_PAGE_HEADER   = Fiddle::SIZEOF_VOIDP
+    SIZEOF_RVALUE        = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
+    HEAP_PAGE_ALIGN_LOG  = 16
+    HEAP_PAGE_ALIGN      = 1 << HEAP_PAGE_ALIGN_LOG      # 2 ^ 14 (or 16 on MacOS)
+    HEAP_PAGE_ALIGN_MASK = ~(~0 << HEAP_PAGE_ALIGN_LOG)  # Mask for getting page address
+    HEAP_PAGE_SIZE       = HEAP_PAGE_ALIGN               # Actual page size
+    HEAP_PAGE_OBJ_LIMIT  = (HEAP_PAGE_SIZE - SIZEOF_PAGE_HEADER) / SIZEOF_RVALUE
+  end
+end

data/lib/heapviz/heap.rb

@@ -0,0 +1,53 @@
+require 'heapviz/config'
+
+module Heapviz
+  class Heap
+    include Heapviz::Config
+
+    def initialize
+      @pages = {}
+    end
+
+    def get_or_build_page_for(slot)
+      @pages[slot.page_body_address] ||= build_page(slot.page_body_address, slot.size)
+    end
+
+    def pages
+      @pages.values
+    end
+
+    def page_count
+      @pages.length
+    end
+
+    def max_page_size
+      HEAP_PAGE_OBJ_LIMIT
+    end
+
+    private
+
+    def build_page(page_body_address, slot_size)
+      # Pages have a header with information, so we have to take that in to account
+      start = page_body_address + SIZEOF_PAGE_HEADER
+
+      # If the object start address isn't evenly divisible by the size of a
+      # Ruby object, we need to calculate the padding required to find the first
+      # address that is divisible by SIZEOF_RVALUE
+      if start % slot_size != 0
+        delta = SIZEOF_RVALUE - (start % SIZEOF_RVALUE)
+        start += delta # Move forward to first address
+
+        if num_in_page(start) == 1
+          start += slot_size - SIZEOF_RVALUE
+        end
+      end
+      limit = (HEAP_PAGE_SIZE - (start - page_body_address)) / slot_size
+
+      Page.new(page_body_address, start, limit, slot_size)
+    end
+
+    def num_in_page(obj)
+      (obj & HEAP_PAGE_ALIGN_MASK) / SIZEOF_RVALUE 
+    end
+  end
+end

data/lib/heapviz/page.rb

@@ -0,0 +1,58 @@
+module Heapviz
+  class Page
+    attr_reader :live_objects, :slot_size, :address
+
+    def initialize(address, obj_start_address, capacity, slot_size)
+      @address = address
+      @obj_start_address = obj_start_address
+      @capacity = capacity
+      @slot_size = slot_size
+
+      @live_objects = []
+    end
+
+    def fill_slot(slot)
+      fail "slot size mismatch" if slot.size != @slot_size
+      @live_objects << slot
+    end
+
+    def each_slot
+      return enum_for(:each_slot) unless block_given?
+
+      objs = sorted_objects
+
+      @capacity.times do |i|
+        expected = @obj_start_address + (i * slot_size)
+        if objs.any? && objs.first.address == expected
+          yield objs.shift
+        else
+          yield nil
+        end
+      end
+    end
+
+    def sorted_objects
+      @live_objects.sort
+    end
+
+    def full?
+      @live_objects.count == @capacity
+    end
+
+    def pinned_count
+      @pinned_count ||= live_objects.find_all { |lo| lo.dig("flags", "pinned") }.count
+    end
+
+    def live_object_count
+      @live_objects.count
+    end
+
+    def fragmented?
+      !full?
+    end
+
+    def to_s
+      "{page: #{address.to_s(16)}, slot_size: #{slot_size}, full: #{full?}}"
+    end
+  end
+end

data/lib/heapviz/renderer.rb

@@ -0,0 +1,68 @@
+require 'logger'
+require 'chunky_png'
+
+module Heapviz
+  class Renderer
+    SLOT_BASE_SIZE = 4
+    COLORS = {
+      40 => ChunkyPNG::Color.rgba(104, 195, 163, 255),
+      80 => ChunkyPNG::Color.rgba(27, 163, 156, 255),
+      160 => ChunkyPNG::Color.rgba(200, 247, 197, 255),
+      320 => ChunkyPNG::Color.rgba(22, 160, 133, 255),
+      640 => ChunkyPNG::Color.rgba(22, 160, 255, 255),
+    }
+    WHITE = ChunkyPNG::Color.rgba(255, 255, 255, 255);
+    BLACK = ChunkyPNG::Color.rgba(0, 0, 0, 255);
+
+    def initialize(op_path, heap, logger = nil)
+      @op_path = op_path
+      @logger = logger || Logger.new(op_path + ".log")
+      @logger.datetime_format = "%s.%L"
+      @heap = heap
+      @img = ChunkyPNG::Image.new(
+        heap.page_count * SLOT_BASE_SIZE,
+        heap.max_page_size * SLOT_BASE_SIZE
+      )
+    end
+
+    def render
+      @heap.pages.each_with_index do |page, i|
+        start_i = i * SLOT_BASE_SIZE
+        render_page(page, start_i)
+      end
+
+      @img.save(@op_path, interlace: true)
+    end
+
+    def slot_colour(slot)
+      return WHITE unless slot
+
+      if slot.pinned?
+        BLACK
+      elsif slot.type == "NONE"
+        WHITE
+      else
+        COLORS[slot.size]
+      end
+    end
+
+    def render_page(page, x)
+      # The height of each slot will be scaled up depending on the
+      # slot size of the page. Slot size is a multiple of rvalue size
+      # right now
+      adjusted_height = SLOT_BASE_SIZE * (page.slot_size / Heap::SIZEOF_RVALUE)
+
+      @logger.debug "#{page}"
+      page.each_slot.with_index do |slot, y|
+        @logger.debug "\t=> #{slot}"
+        y = y * adjusted_height
+
+        adjusted_height.times do |y_offset|
+          SLOT_BASE_SIZE.times do |x_offset|
+            @img[x + x_offset, y + y_offset] = slot_colour(slot)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/heapviz/slot.rb

@@ -0,0 +1,43 @@
+require 'json'
+require 'heapviz/heap'
+
+module Heapviz
+  class Slot
+    include Comparable
+
+    attr_reader :type, :address, :flags, :size, :page_body_address
+
+    def initialize(json)
+      attrs = JSON.parse(json)
+
+      @type = attrs.fetch("type")
+      if !gc_root?
+          @address  = attrs.fetch("address").to_i(16)
+          @size     = attrs.fetch("slot_size")
+      end
+      @flags    = attrs.fetch("flags", {})
+
+      @page_body_address = @address & ~Heap::HEAP_PAGE_ALIGN_MASK
+    end
+
+    def gc_root?
+      @type == "ROOT"
+    end
+
+    def pinned?
+      @flags.fetch('pinned', false)
+    end
+
+    def has_flags?
+      !gc_root?
+    end
+
+    def <=>(o)
+      address <=> o.address
+    end
+
+    def to_s
+      "{address: 0x#{address.to_s(16)}, type: #{type}, flags: #{flags}}"
+    end
+  end
+end

data/lib/heapviz/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Heapviz
+  VERSION = "0.1.0"
+end

data/lib/heapviz.rb

@@ -0,0 +1,7 @@
+require 'fiddle'
+require 'heapviz/version.rb'
+
+require 'heapviz/heap'
+require 'heapviz/page'
+require 'heapviz/slot'
+require 'heapviz/renderer'

data/test.rb

@@ -0,0 +1,14 @@
+def popcount(n)
+  x = 0
+  while n > 1
+    x += 1
+    n >>= 1
+    p n.to_s(2)
+  end
+  x
+end
+
+
+p popcount 0b111000 
+p popcount 0b111111
+

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: heapviz
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matt Valentine-House
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chunky_png
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+description: Visualise a Ruby heap dump output from ObjectSpace.dump_all
+email:
+- matt@eightbitraptor.com
+executables:
+- heapviz
+extensions: []
+extra_rdoc_files: []
+files:
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/heapviz
+- lib/heapviz.rb
+- lib/heapviz/config.rb
+- lib/heapviz/heap.rb
+- lib/heapviz/page.rb
+- lib/heapviz/renderer.rb
+- lib/heapviz/slot.rb
+- lib/heapviz/version.rb
+- test.rb
+homepage: https://github.com/eightbitraptor/heapviz
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/eightbitraptor/heapviz
+  source_code_uri: https://github.com/eightbitraptor/heapviz
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: Visualise a Ruby heap dump output from ObjectSpace.dump_all
+test_files: []