heap-profiler 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -143,12 +143,13 @@ module HeapProfiler
143
143
  end
144
144
  end
145
145
 
146
- def top_n(_max)
146
+ def top_n(max)
147
147
  values = @locations_counts.values
148
148
  values.sort! do |a, b|
149
149
  cmp = b.count <=> a.count
150
150
  cmp == 0 ? b.location <=> a.location : cmp
151
151
  end
152
+ values.take(max)
152
153
  end
153
154
  end
154
155
 
@@ -178,6 +179,29 @@ module HeapProfiler
178
179
  end
179
180
  end
180
181
 
182
+ class ShapeEdgeDimension
183
+ def initialize
184
+ @stats = Hash.new(0)
185
+ end
186
+
187
+ def process(_index, object)
188
+ if name = object[:edge_name]
189
+ @stats[name] += 1
190
+ end
191
+ end
192
+
193
+ def top_n(max)
194
+ @stats.sort do |(a_name, a_count), (b_name, b_count)|
195
+ cmp = b_count <=> a_count
196
+ if cmp == 0
197
+ a_name <=> b_name
198
+ else
199
+ cmp
200
+ end
201
+ end.take(max)
202
+ end
203
+ end
204
+
181
205
  def initialize(heap, index)
182
206
  @heap = heap
183
207
  @index = index
@@ -188,6 +212,8 @@ module HeapProfiler
188
212
  metrics.each do |metric|
189
213
  if metric == "strings"
190
214
  dimensions["strings"] = StringDimension.new
215
+ elsif metric == "shape_edges"
216
+ dimensions["shape_edges"] = ShapeEdgeDimension.new
191
217
  else
192
218
  dimensions["total"] = Dimension.new
193
219
  groupings.each do |grouping|
@@ -11,9 +11,18 @@ module HeapProfiler
11
11
  parser.parse!(@argv)
12
12
 
13
13
  begin
14
- if @argv.size == 1
15
- print_report(@argv.first)
14
+ case @argv.first
15
+ when "clean"
16
+ clean_dump(@argv[1])
16
17
  return 0
18
+ when "report"
19
+ print_report(@argv[1])
20
+ return 0
21
+ else
22
+ if @argv.size == 1
23
+ print_report(@argv.first)
24
+ return 0
25
+ end
17
26
  end
18
27
  rescue CapacityError => error
19
28
  STDERR.puts(error.message)
@@ -27,15 +36,43 @@ module HeapProfiler
27
36
 
28
37
  def print_report(path)
29
38
  results = if File.directory?(path)
30
- DiffResults.new(path)
39
+ if @retained_only
40
+ DiffResults.new(path, ["retained"])
41
+ else
42
+ DiffResults.new(path)
43
+ end
31
44
  else
32
45
  HeapResults.new(path)
33
46
  end
34
47
  results.pretty_print(scale_bytes: true, normalize_paths: true)
35
48
  end
36
49
 
50
+ def clean_dump(path)
51
+ require "json"
52
+ errors = index = 0
53
+ clean_path = "#{path}.clean"
54
+ File.open(clean_path, "w+") do |output|
55
+ File.open(path) do |input|
56
+ input.each_line do |line|
57
+ begin
58
+ JSON.parse(line)
59
+ rescue JSON::ParserError
60
+ errors += 1
61
+ $stderr.puts("Invalid JSON found on line #{index}. Skipping")
62
+ else
63
+ output.print(line)
64
+ end
65
+ index += 1
66
+ end
67
+ end
68
+ end
69
+ $stderr.puts("Processed #{index} lines, removed #{errors} invalid lines")
70
+ $stderr.puts("Clean dump available at #{clean_path}")
71
+ end
72
+
37
73
  def print_usage
38
74
  puts "Usage: #{$PROGRAM_NAME} directory_or_heap_dump"
75
+ puts @parser.help
39
76
  end
40
77
 
41
78
  SIZE_UNITS = {
@@ -66,14 +103,30 @@ module HeapProfiler
66
103
 
67
104
  def parser
68
105
  @parser ||= OptionParser.new do |opts|
69
- opts.banner = "Usage: heap-profiler [ARGS]"
70
- opts.separator ""
71
- opts.separator "GLOBAL OPTIONS"
106
+ opts.banner = <<~EOS
107
+ Usage: heap-profiler [SUBCOMMAND] [ARGS]"
108
+
109
+ SUBCOMMANDS
110
+
111
+ report: Produce a full memory report from the provided dump. (default)
112
+
113
+ clean: Remove all malformed lines from the provided heap dump. Can be useful to workaround some ruby bugs.
114
+
115
+ GLOBAL OPTIONS
116
+ EOS
72
117
  opts.separator ""
73
118
 
74
- help = <<~EOS
75
- Sets the simdjson parser batch size. It must be larger than the largest JSON document in the
76
- heap dump, and defaults to 10MB.
119
+ opts.on('-r', '--retained-only', 'Only compute report for memory retentions.') do
120
+ @retained_only = true
121
+ end
122
+
123
+ HeapProfiler::AbstractResults.top_entries_count = 50
124
+ opts.on("-m", "--max=NUM", Integer, "Max number of entries to output. (Defaults to 50)") do |arg|
125
+ HeapProfiler::AbstractResults.top_entries_count = arg
126
+ end
127
+
128
+ help = <<~EOS.lines.join(" ")
129
+ Sets the simdjson parser batch size. It must be larger than the largest JSON document in the heap dump, and defaults to 10MB.
77
130
  EOS
78
131
  opts.on('--batch-size SIZE', help.strip) do |size_string|
79
132
  HeapProfiler::Parser.batch_size = parse_byte_size(size_string)
@@ -19,6 +19,7 @@ module HeapProfiler
19
19
  @report_directory = report_directory
20
20
  @allocated = open_dump('allocated')
21
21
  @generation = Integer(File.read(File.join(report_directory, 'generation.info')))
22
+ @generation = nil if @generation == 0
22
23
  end
23
24
 
24
25
  def allocated_diff
@@ -31,6 +31,7 @@ module HeapProfiler
31
31
  REGEXP: "Regexp",
32
32
  MATCH: "MatchData",
33
33
  ROOT: "<VM Root>",
34
+ SHAPE: "SHAPE",
34
35
  }.freeze
35
36
 
36
37
  IMEMO_TYPES = Hash.new { |h, k| h[k] = "<#{k || 'unknown'}> (IMEMO)" }
@@ -35,9 +35,11 @@ module HeapProfiler
35
35
  @dir_path = dir_path
36
36
  @enable_tracing = !allocation_tracing_enabled?
37
37
  @generation = nil
38
+ @partial = true
38
39
  end
39
40
 
40
- def start
41
+ def start(partial: true)
42
+ @partial = partial
41
43
  FileUtils.mkdir_p(@dir_path)
42
44
  ObjectSpace.trace_object_allocations_start if @enable_tracing
43
45
 
@@ -46,7 +48,7 @@ module HeapProfiler
46
48
 
47
49
  HeapProfiler.name_anonymous_modules!
48
50
 
49
- 4.times { GC.start }
51
+ GC.start
50
52
  GC.disable
51
53
  @generation = GC.count
52
54
  end
@@ -60,11 +62,11 @@ module HeapProfiler
60
62
  dump_heap(@allocated_heap)
61
63
 
62
64
  GC.enable
63
- 4.times { GC.start }
64
- dump_heap(@retained_heap, partial: true)
65
+ GC.start
66
+ dump_heap(@retained_heap, partial: @partial)
65
67
  @allocated_heap.close
66
68
  @retained_heap.close
67
- write_info("generation", @generation.to_s)
69
+ write_info("generation", @partial ? @generation.to_s : "0")
68
70
  end
69
71
 
70
72
  def run
@@ -14,11 +14,17 @@ module HeapProfiler
14
14
  24 => 'YB',
15
15
  }.freeze
16
16
 
17
- METRICS = ["memory", "objects", "strings"].freeze
17
+ METRICS = ["memory", "objects", "strings", "shape_edges"].freeze
18
+ GROUPED_METRICS = ["memory", "objects"]
18
19
  GROUPINGS = ["gem", "file", "location", "class"].freeze
19
20
 
20
21
  attr_reader :types, :dimensions
21
22
 
23
+ @top_entries_count = 50
24
+ class << self
25
+ attr_accessor :top_entries_count
26
+ end
27
+
22
28
  def initialize(*, **)
23
29
  raise NotImplementedError
24
30
  end
@@ -78,11 +84,13 @@ module HeapProfiler
78
84
  analyzer = Analyzer.new(heap, index)
79
85
  dimensions = analyzer.run(@metrics, @groupings)
80
86
 
81
- io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
82
- "(#{dimensions['total'].objects} objects)"
87
+ if dimensions['total']
88
+ io.puts "Total: #{scale_bytes(dimensions['total'].memory)} " \
89
+ "(#{dimensions['total'].objects} objects)"
90
+ end
83
91
 
84
92
  @metrics.each do |metric|
85
- next if metric == "strings"
93
+ next unless GROUPED_METRICS.include?(metric)
86
94
  @groupings.each do |grouping|
87
95
  dump_data(io, dimensions, metric, grouping, options)
88
96
  end
@@ -91,11 +99,15 @@ module HeapProfiler
91
99
  if @metrics.include?("strings")
92
100
  dump_strings(io, dimensions, options)
93
101
  end
102
+
103
+ if @metrics.include?("shape_edges")
104
+ dump_shape_edges(io, dimensions, options)
105
+ end
94
106
  end
95
107
 
96
108
  def dump_data(io, dimensions, metric, grouping, options)
97
109
  print_title io, "#{metric} by #{grouping}"
98
- data = dimensions[grouping].top_n(metric, options.fetch(:top, 50))
110
+ data = dimensions[grouping].top_n(metric, AbstractResults.top_entries_count)
99
111
 
100
112
  scale_data = metric == "memory" && options[:scale_bytes]
101
113
  normalize_paths = options[:normalize_paths]
@@ -112,7 +124,7 @@ module HeapProfiler
112
124
  def dump_strings(io, dimensions, options)
113
125
  normalize_paths = options[:normalize_paths]
114
126
  scale_data = options[:scale_bytes]
115
- top = options.fetch(:top, 50)
127
+ top = AbstractResults.top_entries_count
116
128
 
117
129
  print_title(io, "String Report")
118
130
 
@@ -127,6 +139,19 @@ module HeapProfiler
127
139
  io.puts
128
140
  end
129
141
  end
142
+
143
+ def dump_shape_edges(io, dimensions, _options)
144
+ top = AbstractResults.top_entries_count
145
+
146
+ data = dimensions["shape_edges"].top_n(top)
147
+ unless data.empty?
148
+ print_title(io, "Shape Edges Report")
149
+
150
+ data.each do |edge_name, count|
151
+ print_output io, count, edge_name
152
+ end
153
+ end
154
+ end
130
155
  end
131
156
 
132
157
  class DiffResults < AbstractResults
@@ -160,7 +185,7 @@ module HeapProfiler
160
185
 
161
186
  @types.each do |type|
162
187
  @metrics.each do |metric|
163
- next if metric == "strings"
188
+ next unless GROUPED_METRICS.include?(metric)
164
189
  @groupings.each do |grouping|
165
190
  dump_data(io, dimensions, type, metric, grouping, options)
166
191
  end
@@ -172,11 +197,17 @@ module HeapProfiler
172
197
  dump_strings(io, dimensions[type], type, options)
173
198
  end
174
199
  end
200
+
201
+ if @metrics.include?("shape_edges")
202
+ @types.each do |type|
203
+ dump_shape_edges(io, dimensions[type], type, options)
204
+ end
205
+ end
175
206
  end
176
207
 
177
208
  def dump_data(io, dimensions, type, metric, grouping, options)
178
209
  print_title io, "#{type} #{metric} by #{grouping}"
179
- data = dimensions[type][grouping].top_n(metric, options.fetch(:top, 50))
210
+ data = dimensions[type][grouping].top_n(metric, AbstractResults.top_entries_count)
180
211
 
181
212
  scale_data = metric == "memory" && options[:scale_bytes]
182
213
  normalize_paths = options[:normalize_paths]
@@ -193,7 +224,7 @@ module HeapProfiler
193
224
  def dump_strings(io, dimensions, type, options)
194
225
  normalize_paths = options[:normalize_paths]
195
226
  scale_data = options[:scale_bytes]
196
- top = options.fetch(:top, 50)
227
+ top = AbstractResults.top_entries_count
197
228
 
198
229
  print_title(io, "#{type.capitalize} String Report")
199
230
 
@@ -208,5 +239,18 @@ module HeapProfiler
208
239
  io.puts
209
240
  end
210
241
  end
242
+
243
+ def dump_shape_edges(io, dimensions, _type, _options)
244
+ top = AbstractResults.top_entries_count
245
+
246
+ data = dimensions["shape_edges"].top_n(top)
247
+ unless data.empty?
248
+ print_title(io, "Shape Edges Report")
249
+
250
+ data.each do |edge_name, count|
251
+ print_output io, count, edge_name
252
+ end
253
+ end
254
+ end
211
255
  end
212
256
  end
@@ -13,10 +13,10 @@ module HeapProfiler
13
13
  class << self
14
14
  attr_accessor :current_reporter
15
15
 
16
- def start(dir)
16
+ def start(dir, **kwargs)
17
17
  return if current_reporter
18
18
  self.current_reporter = Reporter.new(dir)
19
- current_reporter.start
19
+ current_reporter.start(**kwargs)
20
20
  end
21
21
 
22
22
  def stop
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module HeapProfiler
3
- VERSION = "0.5.0"
3
+ VERSION = "0.7.0"
4
4
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: heap-profiler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-05 00:00:00.000000000 Z
11
+ date: 2023-02-27 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Make several heap dumps and summarize allocated, retained memory
14
14
  email:
@@ -22,7 +22,6 @@ files:
22
22
  - ".github/workflows/tests.yml"
23
23
  - ".gitignore"
24
24
  - ".rubocop.yml"
25
- - ".travis.yml"
26
25
  - Gemfile
27
26
  - Gemfile.lock
28
27
  - LICENSE.txt
@@ -79,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
79
78
  - !ruby/object:Gem::Version
80
79
  version: '0'
81
80
  requirements: []
82
- rubygems_version: 3.2.20
81
+ rubygems_version: 3.4.6
83
82
  signing_key:
84
83
  specification_version: 4
85
84
  summary: Ruby heap profiling tool
data/.travis.yml DELETED
@@ -1,6 +0,0 @@
1
- ---
2
- language: ruby
3
- cache: bundler
4
- rvm:
5
- - 2.6.3
6
- before_install: gem install bundler -v 2.1.4