heap-profiler 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +1 -1
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +23 -21
- data/dev.yml +1 -1
- data/ext/heap_profiler/extconf.rb +2 -0
- data/ext/heap_profiler/heap_profiler.cpp +80 -30
- data/ext/heap_profiler/simdjson.cpp +10973 -16643
- data/ext/heap_profiler/simdjson.h +35699 -2420
- data/heap-profiler.gemspec +1 -1
- data/lib/heap_profiler/analyzer.rb +0 -3
- data/lib/heap_profiler/cli.rb +61 -6
- data/lib/heap_profiler/dump.rb +2 -2
- data/lib/heap_profiler/index.rb +12 -7
- data/lib/heap_profiler/parser.rb +22 -7
- data/lib/heap_profiler/reporter.rb +21 -12
- data/lib/heap_profiler/runtime.rb +1 -0
- data/lib/heap_profiler/version.rb +2 -4
- metadata +3 -3
data/heap-profiler.gemspec
CHANGED
@@ -3,7 +3,7 @@ require_relative 'lib/heap_profiler/version'
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "heap-profiler"
|
6
|
-
spec.version =
|
6
|
+
spec.version = HeapProfiler::VERSION
|
7
7
|
spec.authors = ["Jean Boussier"]
|
8
8
|
spec.email = ["jean.boussier@gmail.com"]
|
9
9
|
|
data/lib/heap_profiler/cli.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
require 'optparse'
|
2
3
|
|
3
4
|
module HeapProfiler
|
4
5
|
class CLI
|
@@ -7,13 +8,21 @@ module HeapProfiler
|
|
7
8
|
end
|
8
9
|
|
9
10
|
def run
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
parser.parse!(@argv)
|
12
|
+
|
13
|
+
begin
|
14
|
+
if @argv.size == 1
|
15
|
+
print_report(@argv.first)
|
16
|
+
return 0
|
17
|
+
end
|
18
|
+
rescue CapacityError => error
|
19
|
+
STDERR.puts(error.message)
|
20
|
+
STDERR.puts("Current size: #{Parser.batch_size}B")
|
21
|
+
STDERR.puts("Try increasing it with --batch-size")
|
22
|
+
STDERR.puts
|
16
23
|
end
|
24
|
+
print_usage
|
25
|
+
1
|
17
26
|
end
|
18
27
|
|
19
28
|
def print_report(path)
|
@@ -28,5 +37,51 @@ module HeapProfiler
|
|
28
37
|
def print_usage
|
29
38
|
puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
|
30
39
|
end
|
40
|
+
|
41
|
+
SIZE_UNITS = {
|
42
|
+
'B' => 1,
|
43
|
+
'K' => 1_000,
|
44
|
+
'M' => 1_000_000,
|
45
|
+
'G' => 1_000_000_000,
|
46
|
+
}
|
47
|
+
def parse_byte_size(size_string)
|
48
|
+
if (match = size_string.match(/\A(\d+)(\w)?B?\z/i))
|
49
|
+
digits = Float(match[1])
|
50
|
+
base = 1
|
51
|
+
unit = match[2]&.upcase
|
52
|
+
if unit
|
53
|
+
base = SIZE_UNITS.fetch(unit) { raise ArgumentError, "Unknown size unit: #{unit}" }
|
54
|
+
end
|
55
|
+
size = (digits * base).to_i
|
56
|
+
if size > 4_000_000_000
|
57
|
+
raise ArgumentError, "Batch size can't be bigger than 4G"
|
58
|
+
end
|
59
|
+
size
|
60
|
+
else
|
61
|
+
raise ArgumentError, "#{size_string} is not a valid size"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def parser
|
68
|
+
@parser ||= OptionParser.new do |opts|
|
69
|
+
opts.banner = "Usage: heap-profiler [ARGS]"
|
70
|
+
opts.separator ""
|
71
|
+
opts.separator "GLOBAL OPTIONS"
|
72
|
+
opts.separator ""
|
73
|
+
|
74
|
+
help = <<~EOS
|
75
|
+
Sets the simdjson parser batch size. It must be larger than the largest JSON document in the
|
76
|
+
heap dump, and defaults to 10MB.
|
77
|
+
EOS
|
78
|
+
opts.on('--batch-size SIZE', help.strip) do |size_string|
|
79
|
+
HeapProfiler::Parser.batch_size = parse_byte_size(size_string)
|
80
|
+
rescue ArgumentError => error
|
81
|
+
STDERR.puts "Invalid batch-size: #{error.message}"
|
82
|
+
exit 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
31
86
|
end
|
32
87
|
end
|
data/lib/heap_profiler/dump.rb
CHANGED
data/lib/heap_profiler/index.rb
CHANGED
@@ -34,7 +34,7 @@ module HeapProfiler
|
|
34
34
|
}.freeze
|
35
35
|
|
36
36
|
IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }
|
37
|
-
DATA_TYPES = Hash.new { |h, k| h[k] = "<#{
|
37
|
+
DATA_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (DATA)" }
|
38
38
|
|
39
39
|
def guess_class(object)
|
40
40
|
type = object[:type]
|
@@ -44,14 +44,19 @@ module HeapProfiler
|
|
44
44
|
|
45
45
|
return IMEMO_TYPES[object[:imemo_type]] if type == :IMEMO
|
46
46
|
|
47
|
-
class_address = object[:class]
|
48
|
-
|
47
|
+
class_name = if (class_address = object[:class])
|
48
|
+
@classes.fetch(class_address) do
|
49
|
+
return DATA_TYPES[object[:struct]] if type == :DATA
|
49
50
|
|
50
|
-
|
51
|
-
|
51
|
+
$stderr.puts("WARNING: Couldn't infer class name of: #{object.inspect}")
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
end
|
52
55
|
|
53
|
-
|
54
|
-
|
56
|
+
if type == :DATA && (class_name.nil? || class_name == "Object")
|
57
|
+
DATA_TYPES[object[:struct]]
|
58
|
+
else
|
59
|
+
class_name
|
55
60
|
end
|
56
61
|
end
|
57
62
|
|
data/lib/heap_profiler/parser.rb
CHANGED
@@ -2,18 +2,33 @@
|
|
2
2
|
|
3
3
|
module HeapProfiler
|
4
4
|
module Parser
|
5
|
+
CLASS_DEFAULT_PROC = ->(_hash, key) { "<Class#0x#{key.to_s(16)}>" }
|
6
|
+
|
7
|
+
class << self
|
8
|
+
attr_accessor :batch_size
|
9
|
+
end
|
10
|
+
self.batch_size = 10_000_000 # 10MB
|
11
|
+
|
5
12
|
class Ruby
|
6
13
|
def build_index(path)
|
7
14
|
require 'json'
|
8
15
|
classes_index = {}
|
16
|
+
classes_index.default_proc = CLASS_DEFAULT_PROC
|
9
17
|
strings_index = {}
|
10
18
|
|
11
19
|
File.open(path).each_line do |line|
|
12
20
|
object = JSON.parse(line, symbolize_names: true)
|
13
21
|
case object[:type]
|
14
22
|
when 'MODULE', 'CLASS'
|
15
|
-
|
16
|
-
|
23
|
+
address = parse_address(object[:address])
|
24
|
+
|
25
|
+
name = object[:name]
|
26
|
+
name ||= if object[:file] && object[:line]
|
27
|
+
"<Class #{object[:file]}:#{object[:line]}>"
|
28
|
+
end
|
29
|
+
|
30
|
+
if name
|
31
|
+
classes_index[address] = name
|
17
32
|
end
|
18
33
|
when 'STRING'
|
19
34
|
next if object[:shared]
|
@@ -32,13 +47,13 @@ module HeapProfiler
|
|
32
47
|
end
|
33
48
|
|
34
49
|
class Native
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
50
|
+
def build_index(path, batch_size: Parser.batch_size)
|
51
|
+
indexes = _build_index(path, batch_size)
|
52
|
+
indexes.first.default_proc = CLASS_DEFAULT_PROC
|
53
|
+
indexes
|
39
54
|
end
|
40
55
|
|
41
|
-
def load_many(path, since: nil, batch_size:
|
56
|
+
def load_many(path, since: nil, batch_size: Parser.batch_size, &block)
|
42
57
|
_load_many(path, since, batch_size, &block)
|
43
58
|
end
|
44
59
|
end
|
@@ -9,8 +9,6 @@ module HeapProfiler
|
|
9
9
|
# So we name them at the start of the profile to avoid that.
|
10
10
|
#
|
11
11
|
# See: https://github.com/ruby/ruby/pull/3349
|
12
|
-
#
|
13
|
-
# TODO: Could we actually do the dump ourselves? objspace is a extension already.
|
14
12
|
if RUBY_VERSION < '2.8'
|
15
13
|
def name_anonymous_modules!
|
16
14
|
ObjectSpace.each_object(Module) do |mod|
|
@@ -56,10 +54,14 @@ module HeapProfiler
|
|
56
54
|
def stop
|
57
55
|
HeapProfiler.name_anonymous_modules!
|
58
56
|
ObjectSpace.trace_object_allocations_stop if @enable_tracing
|
57
|
+
|
58
|
+
# we can't use partial dump for allocated.heap, because we need old generations
|
59
|
+
# as well to build the classes and strings indexes.
|
59
60
|
dump_heap(@allocated_heap)
|
61
|
+
|
60
62
|
GC.enable
|
61
63
|
4.times { GC.start }
|
62
|
-
dump_heap(@retained_heap)
|
64
|
+
dump_heap(@retained_heap, partial: true)
|
63
65
|
@allocated_heap.close
|
64
66
|
@retained_heap.close
|
65
67
|
write_info("generation", @generation.to_s)
|
@@ -84,17 +86,24 @@ module HeapProfiler
|
|
84
86
|
File.write(File.join(@dir_path, "#{key}.info"), value)
|
85
87
|
end
|
86
88
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
# property.
|
91
|
-
class_eval <<~RUBY, '__hprof', __LINE__
|
92
|
-
# frozen_string_literal: true
|
93
|
-
def dump_heap(file)
|
94
|
-
ObjectSpace.dump_all(output: file)
|
89
|
+
if RUBY_VERSION >= '3.0'
|
90
|
+
def dump_heap(file, partial: false)
|
91
|
+
ObjectSpace.dump_all(output: file, since: partial ? @generation : nil)
|
95
92
|
file.close
|
96
93
|
end
|
97
|
-
|
94
|
+
else
|
95
|
+
# ObjectSpace.dump_all does allocate a few objects in itself (https://bugs.ruby-lang.org/issues/17045)
|
96
|
+
# because of this even en empty block of code will report a handful of allocations.
|
97
|
+
# To filter them more easily we attribute call `dump_all` from a method with a very specific `file`
|
98
|
+
# property.
|
99
|
+
class_eval <<~RUBY, '__hprof', __LINE__
|
100
|
+
# frozen_string_literal: true
|
101
|
+
def dump_heap(file, partial: false)
|
102
|
+
ObjectSpace.dump_all(output: file)
|
103
|
+
file.close
|
104
|
+
end
|
105
|
+
RUBY
|
106
|
+
end
|
98
107
|
|
99
108
|
def open_heap(name)
|
100
109
|
File.open(File.join(@dir_path, "#{name}.heap"), 'w+')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: heap-profiler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jean Boussier
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-08-05 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Make several heap dumps and summarize allocated, retained memory
|
14
14
|
email:
|
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: '0'
|
81
81
|
requirements: []
|
82
|
-
rubygems_version: 3.
|
82
|
+
rubygems_version: 3.2.20
|
83
83
|
signing_key:
|
84
84
|
specification_version: 4
|
85
85
|
summary: Ruby heap profiling tool
|