heap-profiler 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +7 -33
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +4 -2
- data/README.md +15 -1
- data/bin/generate-report +13 -0
- data/ext/heap_profiler/heap_profiler.cpp +12 -3
- data/ext/heap_profiler/simdjson.cpp +3472 -409
- data/ext/heap_profiler/simdjson.h +24123 -33047
- data/lib/heap_profiler/analyzer.rb +27 -1
- data/lib/heap_profiler/cli.rb +62 -9
- data/lib/heap_profiler/diff.rb +1 -0
- data/lib/heap_profiler/index.rb +1 -0
- data/lib/heap_profiler/reporter.rb +7 -5
- data/lib/heap_profiler/results.rb +53 -9
- data/lib/heap_profiler/runtime.rb +2 -2
- data/lib/heap_profiler/version.rb +1 -1
- metadata +3 -4
- data/.travis.yml +0 -6
@@ -143,12 +143,13 @@ module HeapProfiler
|
|
143
143
|
end
|
144
144
|
end
|
145
145
|
|
146
|
-
def top_n(
|
146
|
+
def top_n(max)
|
147
147
|
values = @locations_counts.values
|
148
148
|
values.sort! do |a, b|
|
149
149
|
cmp = b.count <=> a.count
|
150
150
|
cmp == 0 ? b.location <=> a.location : cmp
|
151
151
|
end
|
152
|
+
values.take(max)
|
152
153
|
end
|
153
154
|
end
|
154
155
|
|
@@ -178,6 +179,29 @@ module HeapProfiler
|
|
178
179
|
end
|
179
180
|
end
|
180
181
|
|
182
|
+
class ShapeEdgeDimension
|
183
|
+
def initialize
|
184
|
+
@stats = Hash.new(0)
|
185
|
+
end
|
186
|
+
|
187
|
+
def process(_index, object)
|
188
|
+
if name = object[:edge_name]
|
189
|
+
@stats[name] += 1
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def top_n(max)
|
194
|
+
@stats.sort do |(a_name, a_count), (b_name, b_count)|
|
195
|
+
cmp = b_count <=> a_count
|
196
|
+
if cmp == 0
|
197
|
+
a_name <=> b_name
|
198
|
+
else
|
199
|
+
cmp
|
200
|
+
end
|
201
|
+
end.take(max)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
181
205
|
def initialize(heap, index)
|
182
206
|
@heap = heap
|
183
207
|
@index = index
|
@@ -188,6 +212,8 @@ module HeapProfiler
|
|
188
212
|
metrics.each do |metric|
|
189
213
|
if metric == "strings"
|
190
214
|
dimensions["strings"] = StringDimension.new
|
215
|
+
elsif metric == "shape_edges"
|
216
|
+
dimensions["shape_edges"] = ShapeEdgeDimension.new
|
191
217
|
else
|
192
218
|
dimensions["total"] = Dimension.new
|
193
219
|
groupings.each do |grouping|
|
data/lib/heap_profiler/cli.rb
CHANGED
@@ -11,9 +11,18 @@ module HeapProfiler
|
|
11
11
|
parser.parse!(@argv)
|
12
12
|
|
13
13
|
begin
|
14
|
-
|
15
|
-
|
14
|
+
case @argv.first
|
15
|
+
when "clean"
|
16
|
+
clean_dump(@argv[1])
|
16
17
|
return 0
|
18
|
+
when "report"
|
19
|
+
print_report(@argv[1])
|
20
|
+
return 0
|
21
|
+
else
|
22
|
+
if @argv.size == 1
|
23
|
+
print_report(@argv.first)
|
24
|
+
return 0
|
25
|
+
end
|
17
26
|
end
|
18
27
|
rescue CapacityError => error
|
19
28
|
STDERR.puts(error.message)
|
@@ -27,15 +36,43 @@ module HeapProfiler
|
|
27
36
|
|
28
37
|
def print_report(path)
|
29
38
|
results = if File.directory?(path)
|
30
|
-
|
39
|
+
if @retained_only
|
40
|
+
DiffResults.new(path, ["retained"])
|
41
|
+
else
|
42
|
+
DiffResults.new(path)
|
43
|
+
end
|
31
44
|
else
|
32
45
|
HeapResults.new(path)
|
33
46
|
end
|
34
47
|
results.pretty_print(scale_bytes: true, normalize_paths: true)
|
35
48
|
end
|
36
49
|
|
50
|
+
def clean_dump(path)
|
51
|
+
require "json"
|
52
|
+
errors = index = 0
|
53
|
+
clean_path = "#{path}.clean"
|
54
|
+
File.open(clean_path, "w+") do |output|
|
55
|
+
File.open(path) do |input|
|
56
|
+
input.each_line do |line|
|
57
|
+
begin
|
58
|
+
JSON.parse(line)
|
59
|
+
rescue JSON::ParserError
|
60
|
+
errors += 1
|
61
|
+
$stderr.puts("Invalid JSON found on line #{index}. Skipping")
|
62
|
+
else
|
63
|
+
output.print(line)
|
64
|
+
end
|
65
|
+
index += 1
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
$stderr.puts("Processed #{index} lines, removed #{errors} invalid lines")
|
70
|
+
$stderr.puts("Clean dump available at #{clean_path}")
|
71
|
+
end
|
72
|
+
|
37
73
|
def print_usage
|
38
74
|
puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
|
75
|
+
puts @parser.help
|
39
76
|
end
|
40
77
|
|
41
78
|
SIZE_UNITS = {
|
@@ -66,14 +103,30 @@ module HeapProfiler
|
|
66
103
|
|
67
104
|
def parser
|
68
105
|
@parser ||= OptionParser.new do |opts|
|
69
|
-
opts.banner =
|
70
|
-
|
71
|
-
|
106
|
+
opts.banner = <<~EOS
|
107
|
+
Usage: heap-profiler [SUBCOMMAND] [ARGS]"
|
108
|
+
|
109
|
+
SUBCOMMANDS
|
110
|
+
|
111
|
+
report: Produce a full memory report from the provided dump. (default)
|
112
|
+
|
113
|
+
clean: Remove all malformed lines from the provided heap dump. Can be useful to workaround some ruby bugs.
|
114
|
+
|
115
|
+
GLOBAL OPTIONS
|
116
|
+
EOS
|
72
117
|
opts.separator ""
|
73
118
|
|
74
|
-
|
75
|
-
|
76
|
-
|
119
|
+
opts.on('-r', '--retained-only', 'Only compute report for memory retentions.') do
|
120
|
+
@retained_only = true
|
121
|
+
end
|
122
|
+
|
123
|
+
HeapProfiler::AbstractResults.top_entries_count = 50
|
124
|
+
opts.on("-m", "--max=NUM", Integer, "Max number of entries to output. (Defaults to 50)") do |arg|
|
125
|
+
HeapProfiler::AbstractResults.top_entries_count = arg
|
126
|
+
end
|
127
|
+
|
128
|
+
help = <<~EOS.lines.join(" ")
|
129
|
+
Sets the simdjson parser batch size. It must be larger than the largest JSON document in the heap dump, and defaults to 10MB.
|
77
130
|
EOS
|
78
131
|
opts.on('--batch-size SIZE', help.strip) do |size_string|
|
79
132
|
HeapProfiler::Parser.batch_size = parse_byte_size(size_string)
|
data/lib/heap_profiler/diff.rb
CHANGED
data/lib/heap_profiler/index.rb
CHANGED
@@ -35,9 +35,11 @@ module HeapProfiler
|
|
35
35
|
@dir_path = dir_path
|
36
36
|
@enable_tracing = !allocation_tracing_enabled?
|
37
37
|
@generation = nil
|
38
|
+
@partial = true
|
38
39
|
end
|
39
40
|
|
40
|
-
def start
|
41
|
+
def start(partial: true)
|
42
|
+
@partial = partial
|
41
43
|
FileUtils.mkdir_p(@dir_path)
|
42
44
|
ObjectSpace.trace_object_allocations_start if @enable_tracing
|
43
45
|
|
@@ -46,7 +48,7 @@ module HeapProfiler
|
|
46
48
|
|
47
49
|
HeapProfiler.name_anonymous_modules!
|
48
50
|
|
49
|
-
|
51
|
+
GC.start
|
50
52
|
GC.disable
|
51
53
|
@generation = GC.count
|
52
54
|
end
|
@@ -60,11 +62,11 @@ module HeapProfiler
|
|
60
62
|
dump_heap(@allocated_heap)
|
61
63
|
|
62
64
|
GC.enable
|
63
|
-
|
64
|
-
dump_heap(@retained_heap, partial:
|
65
|
+
GC.start
|
66
|
+
dump_heap(@retained_heap, partial: @partial)
|
65
67
|
@allocated_heap.close
|
66
68
|
@retained_heap.close
|
67
|
-
write_info("generation", @generation.to_s)
|
69
|
+
write_info("generation", @partial ? @generation.to_s : "0")
|
68
70
|
end
|
69
71
|
|
70
72
|
def run
|
@@ -14,11 +14,17 @@ module HeapProfiler
|
|
14
14
|
24 => 'YB',
|
15
15
|
}.freeze
|
16
16
|
|
17
|
-
METRICS = ["memory", "objects", "strings"].freeze
|
17
|
+
METRICS = ["memory", "objects", "strings", "shape_edges"].freeze
|
18
|
+
GROUPED_METRICS = ["memory", "objects"]
|
18
19
|
GROUPINGS = ["gem", "file", "location", "class"].freeze
|
19
20
|
|
20
21
|
attr_reader :types, :dimensions
|
21
22
|
|
23
|
+
@top_entries_count = 50
|
24
|
+
class << self
|
25
|
+
attr_accessor :top_entries_count
|
26
|
+
end
|
27
|
+
|
22
28
|
def initialize(*, **)
|
23
29
|
raise NotImplementedError
|
24
30
|
end
|
@@ -78,11 +84,13 @@ module HeapProfiler
|
|
78
84
|
analyzer = Analyzer.new(heap, index)
|
79
85
|
dimensions = analyzer.run(@metrics, @groupings)
|
80
86
|
|
81
|
-
|
82
|
-
|
87
|
+
if dimensions['total']
|
88
|
+
io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
|
89
|
+
"(#{dimensions['total'].objects} objects)"
|
90
|
+
end
|
83
91
|
|
84
92
|
@metrics.each do |metric|
|
85
|
-
next
|
93
|
+
next unless GROUPED_METRICS.include?(metric)
|
86
94
|
@groupings.each do |grouping|
|
87
95
|
dump_data(io, dimensions, metric, grouping, options)
|
88
96
|
end
|
@@ -91,11 +99,15 @@ module HeapProfiler
|
|
91
99
|
if @metrics.include?("strings")
|
92
100
|
dump_strings(io, dimensions, options)
|
93
101
|
end
|
102
|
+
|
103
|
+
if @metrics.include?("shape_edges")
|
104
|
+
dump_shape_edges(io, dimensions, options)
|
105
|
+
end
|
94
106
|
end
|
95
107
|
|
96
108
|
def dump_data(io, dimensions, metric, grouping, options)
|
97
109
|
print_title io, "#{metric} by #{grouping}"
|
98
|
-
data = dimensions[grouping].top_n(metric,
|
110
|
+
data = dimensions[grouping].top_n(metric, AbstractResults.top_entries_count)
|
99
111
|
|
100
112
|
scale_data = metric == "memory" && options[:scale_bytes]
|
101
113
|
normalize_paths = options[:normalize_paths]
|
@@ -112,7 +124,7 @@ module HeapProfiler
|
|
112
124
|
def dump_strings(io, dimensions, options)
|
113
125
|
normalize_paths = options[:normalize_paths]
|
114
126
|
scale_data = options[:scale_bytes]
|
115
|
-
top =
|
127
|
+
top = AbstractResults.top_entries_count
|
116
128
|
|
117
129
|
print_title(io, "String Report")
|
118
130
|
|
@@ -127,6 +139,19 @@ module HeapProfiler
|
|
127
139
|
io.puts
|
128
140
|
end
|
129
141
|
end
|
142
|
+
|
143
|
+
def dump_shape_edges(io, dimensions, _options)
|
144
|
+
top = AbstractResults.top_entries_count
|
145
|
+
|
146
|
+
data = dimensions["shape_edges"].top_n(top)
|
147
|
+
unless data.empty?
|
148
|
+
print_title(io, "Shape Edges Report")
|
149
|
+
|
150
|
+
data.each do |edge_name, count|
|
151
|
+
print_output io, count, edge_name
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
130
155
|
end
|
131
156
|
|
132
157
|
class DiffResults < AbstractResults
|
@@ -160,7 +185,7 @@ module HeapProfiler
|
|
160
185
|
|
161
186
|
@types.each do |type|
|
162
187
|
@metrics.each do |metric|
|
163
|
-
next
|
188
|
+
next unless GROUPED_METRICS.include?(metric)
|
164
189
|
@groupings.each do |grouping|
|
165
190
|
dump_data(io, dimensions, type, metric, grouping, options)
|
166
191
|
end
|
@@ -172,11 +197,17 @@ module HeapProfiler
|
|
172
197
|
dump_strings(io, dimensions[type], type, options)
|
173
198
|
end
|
174
199
|
end
|
200
|
+
|
201
|
+
if @metrics.include?("shape_edges")
|
202
|
+
@types.each do |type|
|
203
|
+
dump_shape_edges(io, dimensions[type], type, options)
|
204
|
+
end
|
205
|
+
end
|
175
206
|
end
|
176
207
|
|
177
208
|
def dump_data(io, dimensions, type, metric, grouping, options)
|
178
209
|
print_title io, "#{type} #{metric} by #{grouping}"
|
179
|
-
data = dimensions[type][grouping].top_n(metric,
|
210
|
+
data = dimensions[type][grouping].top_n(metric, AbstractResults.top_entries_count)
|
180
211
|
|
181
212
|
scale_data = metric == "memory" && options[:scale_bytes]
|
182
213
|
normalize_paths = options[:normalize_paths]
|
@@ -193,7 +224,7 @@ module HeapProfiler
|
|
193
224
|
def dump_strings(io, dimensions, type, options)
|
194
225
|
normalize_paths = options[:normalize_paths]
|
195
226
|
scale_data = options[:scale_bytes]
|
196
|
-
top =
|
227
|
+
top = AbstractResults.top_entries_count
|
197
228
|
|
198
229
|
print_title(io, "#{type.capitalize} String Report")
|
199
230
|
|
@@ -208,5 +239,18 @@ module HeapProfiler
|
|
208
239
|
io.puts
|
209
240
|
end
|
210
241
|
end
|
242
|
+
|
243
|
+
def dump_shape_edges(io, dimensions, _type, _options)
|
244
|
+
top = AbstractResults.top_entries_count
|
245
|
+
|
246
|
+
data = dimensions["shape_edges"].top_n(top)
|
247
|
+
unless data.empty?
|
248
|
+
print_title(io, "Shape Edges Report")
|
249
|
+
|
250
|
+
data.each do |edge_name, count|
|
251
|
+
print_output io, count, edge_name
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
211
255
|
end
|
212
256
|
end
|
@@ -13,10 +13,10 @@ module HeapProfiler
|
|
13
13
|
class << self
|
14
14
|
attr_accessor :current_reporter
|
15
15
|
|
16
|
-
def start(dir)
|
16
|
+
def start(dir, **kwargs)
|
17
17
|
return if current_reporter
|
18
18
|
self.current_reporter = Reporter.new(dir)
|
19
|
-
current_reporter.start
|
19
|
+
current_reporter.start(**kwargs)
|
20
20
|
end
|
21
21
|
|
22
22
|
def stop
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: heap-profiler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jean Boussier
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Make several heap dumps and summarize allocated, retained memory
|
14
14
|
email:
|
@@ -22,7 +22,6 @@ files:
|
|
22
22
|
- ".github/workflows/tests.yml"
|
23
23
|
- ".gitignore"
|
24
24
|
- ".rubocop.yml"
|
25
|
-
- ".travis.yml"
|
26
25
|
- Gemfile
|
27
26
|
- Gemfile.lock
|
28
27
|
- LICENSE.txt
|
@@ -79,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
79
78
|
- !ruby/object:Gem::Version
|
80
79
|
version: '0'
|
81
80
|
requirements: []
|
82
|
-
rubygems_version: 3.
|
81
|
+
rubygems_version: 3.4.6
|
83
82
|
signing_key:
|
84
83
|
specification_version: 4
|
85
84
|
summary: Ruby heap profiling tool
|