memory 0.0.3 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: dc40099542a524f07fbf5b7366935ea4ec2f9b935df2f6f8016067c1d210794f
4
+ data.tar.gz: 6d5a19607b091081788dbf26bab000a7b1f221befb5c7f9aece9d80f4583fa44
5
+ SHA512:
6
+ metadata.gz: 8e546c7a3d224c4349152630e37c0c00446aff9184395dbdb9699eec0eacfdd9724e313a33c7442b99c6fdcc6a684a75d28d54f99c78122260bda499503f1682
7
+ data.tar.gz: a4760e52fc70250c44faf2685dac5101fe726ef719578cb37dbabe00db4eeebb9cfe46cad41f7238e45d5bd6eb5a591f8f96d7d61766e385567739aad4a49cfa
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ def check
4
+ require 'console'
5
+
6
+ paths = Dir["../../*.mprof"]
7
+
8
+ total_size = paths.sum{|path| File.size(path)}
9
+
10
+ require_relative 'lib/memory_profiler'
11
+
12
+ report = Memory::Report.general
13
+
14
+ cache = Memory::Cache.new
15
+ wrapper = Memory::Wrapper.new(cache)
16
+
17
+ measure = Console.logger.measure(report, total_size)
18
+
19
+ paths.each do |path|
20
+ Console.logger.info(report, "Loading #{path}, #{Memory.formatted_bytes File.size(path)}")
21
+
22
+ File.open(path) do |io|
23
+ unpacker = wrapper.unpacker(io)
24
+ count = unpacker.read_array_header
25
+
26
+ report.concat(unpacker)
27
+
28
+ measure.increment(io.size)
29
+ end
30
+
31
+ Console.logger.info(report, "Loaded allocations, #{report.total_allocated}")
32
+ end
33
+
34
+ report.print($stdout)
35
+
36
+ binding.irb
37
+ end
@@ -1,3 +1,35 @@
1
+ # frozen_string_literal: true
1
2
 
2
- require 'memory/usage'
3
- # require 'memory/profile'
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require_relative "memory/version"
24
+ require_relative "memory/cache"
25
+ require_relative "memory/report"
26
+ require_relative "memory/sampler"
27
+
28
+ module Memory
29
+ def self.report(&block)
30
+ sampler = Sampler.new
31
+ sampler.run(&block)
32
+
33
+ return sampler.report
34
+ end
35
+ end
@@ -0,0 +1,128 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Memory
24
+ UNITS = {
25
+ 0 => 'B',
26
+ 3 => 'KiB',
27
+ 6 => 'MiB',
28
+ 9 => 'GiB',
29
+ 12 => 'TiB',
30
+ 15 => 'PiB',
31
+ 18 => 'EiB',
32
+ 21 => 'ZiB',
33
+ 24 => 'YiB'
34
+ }.freeze
35
+
36
+ def self.formatted_bytes(bytes)
37
+ return "0 B" if bytes.zero?
38
+
39
+ scale = Math.log2(bytes).div(10) * 3
40
+ scale = 24 if scale > 24
41
+ "%.2f #{UNITS[scale]}" % (bytes / 10.0**scale)
42
+ end
43
+
44
+ class Aggregate
45
+ Total = Struct.new(:memory, :count) do
46
+ def initialize
47
+ super(0, 0)
48
+ end
49
+
50
+ def << allocation
51
+ self.memory += allocation.size
52
+ self.count += 1
53
+ end
54
+
55
+ def formatted_memory
56
+ self.memory
57
+ end
58
+
59
+ def to_s
60
+ "(#{Memory.formatted_bytes memory} in #{count} allocations)"
61
+ end
62
+ end
63
+
64
+ def initialize(title, &block)
65
+ @title = title
66
+ @metric = block
67
+
68
+ @total = Total.new
69
+ @totals = Hash.new{|h,k| h[k] = Total.new}
70
+ end
71
+
72
+ attr :total
73
+
74
+ def << allocation
75
+ metric = @metric.call(allocation)
76
+ total = @totals[metric]
77
+
78
+ total.memory += allocation.memsize
79
+ total.count += 1
80
+
81
+ @total.memory += allocation.memsize
82
+ @total.count += 1
83
+ end
84
+
85
+ def totals_by(key)
86
+ @totals.sort_by{|metric, total| [total[key], metric]}
87
+ end
88
+
89
+ def print(io = $stderr, limit: 10, title: @title, level: 2)
90
+ io.puts "#{'#' * level} #{title} #{@total}", nil
91
+
92
+ totals_by(:memory).last(limit).reverse_each do |metric, total|
93
+ io.puts "- #{total}\t#{metric}"
94
+ end
95
+
96
+ io.puts nil
97
+ end
98
+ end
99
+
100
+ class ValueAggregate
101
+ def initialize(title, &block)
102
+ @title = title
103
+ @metric = block
104
+
105
+ @aggregates = Hash.new{|h,k| h[k] = Aggregate.new(k.inspect, &@metric)}
106
+ end
107
+
108
+ def << allocation
109
+ if value = allocation.value
110
+ aggregate = @aggregates[value]
111
+
112
+ aggregate << allocation
113
+ end
114
+ end
115
+
116
+ def aggregates_by(key)
117
+ @aggregates.sort_by{|value, aggregate| [aggregate.total[key], value]}
118
+ end
119
+
120
+ def print(io = $stderr, limit: 10, level: 2)
121
+ io.puts "#{'#' * level} #{@title}", nil
122
+
123
+ aggregates_by(:count).last(limit).reverse_each do |value, aggregate|
124
+ aggregate.print(io, level: level+1)
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Memory
24
+ class Cache
25
+ def initialize
26
+ @gem_guess_cache = Hash.new
27
+ @location_cache = Hash.new { |h, k| h[k] = Hash.new.compare_by_identity }
28
+ @class_name_cache = Hash.new.compare_by_identity
29
+ @string_cache = Hash.new
30
+ end
31
+
32
+ def guess_gem(path)
33
+ @gem_guess_cache[path] ||=
34
+ if /(\/gems\/.*)*\/gems\/(?<gemname>[^\/]+)/ =~ path
35
+ gemname
36
+ elsif /\/rubygems[\.\/]/ =~ path
37
+ "rubygems"
38
+ elsif /ruby\/2\.[^\/]+\/(?<stdlib>[^\/\.]+)/ =~ path
39
+ stdlib
40
+ elsif /(?<app>[^\/]+\/(bin|app|lib))/ =~ path
41
+ app
42
+ else
43
+ "other"
44
+ end
45
+ end
46
+
47
+ def lookup_location(file, line)
48
+ @location_cache[file][line] ||= "#{file}:#{line}"
49
+ end
50
+
51
+ def lookup_class_name(klass)
52
+ @class_name_cache[klass] ||= ((klass.is_a?(Class) && klass.name) || '<<Unknown>>').to_s
53
+ end
54
+
55
+ def lookup_string(obj)
56
+ # This string is shortened to 200 characters which is what the string report shows
57
+ # The string report can still list unique strings longer than 200 characters
58
+ # separately because the object_id of the shortened string will be different
59
+ @string_cache[obj] ||= String.new << obj[0, 64]
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'objspace'
24
+ require 'msgpack'
25
+
26
+ module Memory
27
+ class Deque
28
+ def initialize
29
+ @segments = []
30
+ @last = nil
31
+ end
32
+
33
+ def freeze
34
+ return self if frozen?
35
+
36
+ @segments.each(&:freeze)
37
+ @last = nil
38
+
39
+ super
40
+ end
41
+
42
+ include Enumerable
43
+
44
+ def concat(segment)
45
+ @segments << segment
46
+ @last = nil
47
+
48
+ return self
49
+ end
50
+
51
+ def << item
52
+ unless @last
53
+ @last = []
54
+ @segments << @last
55
+ end
56
+
57
+ @last << item
58
+
59
+ return self
60
+ end
61
+
62
+ def each(&block)
63
+ @segments.each do |segment|
64
+ segment.each(&block)
65
+ end
66
+ end
67
+
68
+ def size
69
+ @segments.sum(&:size)
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require_relative 'aggregate'
24
+
25
+ module Memory
26
+ class Report
27
+ def self.general
28
+ Report.new([
29
+ Aggregate.new("By Gem", &:gem),
30
+ Aggregate.new("By File", &:file),
31
+ Aggregate.new("By Location", &:location),
32
+ Aggregate.new("By Class", &:class_name),
33
+ ValueAggregate.new("Strings By Gem", &:gem),
34
+ ValueAggregate.new("Strings By Location", &:location),
35
+ ])
36
+ end
37
+
38
+ def initialize(aggregates)
39
+ @total_allocated = Aggregate::Total.new
40
+ @total_retained = Aggregate::Total.new
41
+
42
+ @aggregates = aggregates
43
+ end
44
+
45
+ attr :total_allocated
46
+
47
+ def concat(allocations)
48
+ allocations.each do |allocation|
49
+ @total_allocated << allocation
50
+ @total_retained << allocation if allocation.retained
51
+
52
+ @aggregates.each do |aggregate|
53
+ aggregate << allocation
54
+ end
55
+ end
56
+ end
57
+
58
+ def print(io = $stderr)
59
+ io.puts "\# Memory Profile", nil
60
+
61
+ io.puts "- Total Allocated: #{@total_allocated}"
62
+ io.puts "- Total Retained: #{@total_retained}"
63
+ io.puts
64
+
65
+ @aggregates.each do |aggregate|
66
+ aggregate.print(io)
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'objspace'
24
+ require 'msgpack'
25
+ require 'console'
26
+
27
+ module Memory
28
+ class Wrapper < MessagePack::Factory
29
+ def initialize(cache)
30
+ super()
31
+
32
+ @cache = cache
33
+
34
+ self.register_type(0x01, Allocation,
35
+ packer: ->(instance){self.pack(instance.pack)},
36
+ unpacker: ->(data){Allocation.unpack(@cache, self.unpack(data))},
37
+ )
38
+
39
+ self.register_type(0x02, Symbol)
40
+ end
41
+ end
42
+
43
+ Allocation = Struct.new(:cache, :class_name, :file, :line, :memsize, :value, :retained) do
44
+ def location
45
+ cache.lookup_location(file, line)
46
+ end
47
+
48
+ def gem
49
+ cache.guess_gem(file)
50
+ end
51
+
52
+ def pack
53
+ [class_name, file, line, memsize, value, retained]
54
+ end
55
+
56
+ def self.unpack(cache, fields)
57
+ self.new(cache, *fields)
58
+ end
59
+ end
60
+
61
+ # Sample memory allocations.
62
+ #
63
+ # ~~~ ruby
64
+ # sampler = Sampler.capture do
65
+ # 5.times { "foo" }
66
+ # end
67
+ # ~~~
68
+ class Sampler
69
+ def initialize(&filter)
70
+ @filter = filter
71
+
72
+ @cache = Cache.new
73
+ @wrapper = Wrapper.new(@cache)
74
+ @allocated = Array.new
75
+ end
76
+
77
+ attr :filter
78
+
79
+ attr :cache
80
+ attr :wrapper
81
+ attr :allocated
82
+
83
+ def start
84
+ GC.disable
85
+ GC.start
86
+
87
+ @generation = GC.count
88
+ ObjectSpace.trace_object_allocations_start
89
+ end
90
+
91
+ def stop
92
+ ObjectSpace.trace_object_allocations_stop
93
+ allocated = track_allocations(@generation)
94
+
95
+ # **WARNING** Do not allocate any new Objects between the call to GC.start and the completion of the retained lookups. It is likely that a new Object would reuse an object_id from a GC'd object.
96
+
97
+ GC.enable
98
+ 3.times{GC.start}
99
+
100
+ ObjectSpace.each_object do |object|
101
+ next unless ObjectSpace.allocation_generation(object) == @generation
102
+
103
+ if found = allocated[object.__id__]
104
+ found.retained = true
105
+ end
106
+ end
107
+
108
+ ObjectSpace.trace_object_allocations_clear
109
+ end
110
+
111
+ def dump(io = nil)
112
+ Console.logger.debug(self, "Dumping allocations: #{@allocated.size}")
113
+
114
+ if io
115
+ packer = @wrapper.packer(io)
116
+ packer.pack(@allocated)
117
+ packer.flush
118
+ else
119
+ @wrapper.dump(@allocated)
120
+ end
121
+ end
122
+
123
+ def load(data)
124
+ allocations = @wrapper.load(data)
125
+
126
+ Console.logger.debug(self, "Loading allocations: #{allocations.size}")
127
+
128
+ @allocated.concat(allocations)
129
+ end
130
+
131
+ def report
132
+ report = Report.general
133
+
134
+ report.concat(@allocated)
135
+
136
+ return report
137
+ end
138
+
139
+ # Collects object allocation and memory of ruby code inside of passed block.
140
+ def run(&block)
141
+ start
142
+
143
+ begin
144
+ # We do this to avoid retaining the result of the block.
145
+ yield && false
146
+ ensure
147
+ stop
148
+ end
149
+ end
150
+
151
+ private
152
+
153
+ # Iterates through objects in memory of a given generation.
154
+ # Stores results along with meta data of objects collected.
155
+ def track_allocations(generation)
156
+ rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
157
+
158
+ allocated = Hash.new.compare_by_identity
159
+
160
+ ObjectSpace.each_object do |object|
161
+ next unless ObjectSpace.allocation_generation(object) == generation
162
+
163
+ file = ObjectSpace.allocation_sourcefile(object) || "(no name)"
164
+
165
+ klass = object.class rescue nil
166
+
167
+ unless Class === klass
168
+ # attempt to determine the true Class when .class returns something other than a Class
169
+ klass = Kernel.instance_method(:class).bind(object).call
170
+ end
171
+
172
+ next if @filter && !@filter.call(klass, file)
173
+
174
+ line = ObjectSpace.allocation_sourceline(object)
175
+
176
+ # we do memsize first to avoid freezing as a side effect and shifting
177
+ # storage to the new frozen string, this happens on @hash[s] in lookup_string
178
+ memsize = ObjectSpace.memsize_of(object)
179
+ class_name = @cache.lookup_class_name(klass)
180
+ value = (klass == String) ? @cache.lookup_string(object) : nil
181
+
182
+ # compensate for API bug
183
+ memsize = rvalue_size if memsize > 100_000_000_000
184
+
185
+ allocation = Allocation.new(@cache, class_name, file, line, memsize, value, false)
186
+
187
+ @allocated << allocation
188
+ allocated[object.__id__] = allocation
189
+ end
190
+
191
+ return allocated
192
+ end
193
+ end
194
+ end