awfy 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +182 -10
- data/lib/awfy/cli.rb +195 -57
- data/lib/awfy/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f9e397437b1d2fddfcad8796896160185c10f1832f98468e7b828b4e7e717a9e
|
|
4
|
+
data.tar.gz: 055a0bb06e33dcd7b77bd1231b1adce331f8a7e2966a0bdf014bd597d12bea2f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fa51eab588e804c99b061401cc906ed58bbee83aae735839dba54eabc1a61800a99bcce7c613aeaa5b1e80c73906939cf93b4c17cf5feba54af2b3faf4e5556a
|
|
7
|
+
data.tar.gz: 6012ad3784a7eb769d25d47d88df7ff1bc19e0a3c5390d12c2a7f447c407c348b61c9bfd42dcef3fcb6ec7dbaf9acf2368c56afdf30cb1f91f8fc74591e54f3c
|
data/README.md
CHANGED
|
@@ -1,38 +1,210 @@
|
|
|
1
1
|
# Awfy (Are We Fast Yet)
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
CLI tool to help run suites of benchmarks and compare results between control implementations, across branches and with or without YJIT.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
The benchmarks are written using a simple DSL in your target project.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Supports running:
|
|
8
|
+
|
|
9
|
+
- IPS benchmarks (with [benchmark-ips](https://rubygems.org/gems/benchmark-ips))
|
|
10
|
+
- Memory profiling (with [memory_profiler](https://rubygems.org/gems/memory_profiler))
|
|
11
|
+
- CPU profiling (with [stackprof](https://rubygems.org/gems/stackprof))
|
|
12
|
+
- Flamegraph profiling (with [singed](https://rubygems.org/gems/singed))
|
|
8
13
|
|
|
9
|
-
|
|
14
|
+
Awfy can also create summary reports of the results which can be useful for comparing the performance of different implementations **(currently only supported for IPS benchmarks)**.
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
10
17
|
|
|
11
18
|
Install the gem and add to the application's Gemfile by executing:
|
|
12
19
|
|
|
13
20
|
```bash
|
|
14
|
-
bundle add
|
|
21
|
+
bundle add awfy
|
|
15
22
|
```
|
|
16
23
|
|
|
17
24
|
If bundler is not being used to manage dependencies, install the gem by executing:
|
|
18
25
|
|
|
19
26
|
```bash
|
|
20
|
-
gem install
|
|
27
|
+
gem install awfy
|
|
21
28
|
```
|
|
22
29
|
|
|
23
30
|
## Usage
|
|
24
31
|
|
|
25
|
-
|
|
32
|
+
Imagine we have a custom implementation of a Struct class called `MyStruct`. We want to compare the performance of our implementation with the built-in `Struct` class
|
|
33
|
+
and other similar implementations.
|
|
34
|
+
|
|
35
|
+
First, we need to create a setup file in the `benchmarks/setup.rb` directory. For example:
|
|
36
|
+
|
|
37
|
+
```ruby
|
|
38
|
+
# setup.rb
|
|
39
|
+
|
|
40
|
+
require "dry-struct"
|
|
41
|
+
require "active_model"
|
|
42
|
+
|
|
43
|
+
class DryStruct < Dry::Struct
|
|
44
|
+
attribute :name, Types::String
|
|
45
|
+
attribute :age, Types::Integer
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
class ActiveModelAttributes
|
|
49
|
+
include ActiveModel::API
|
|
50
|
+
include ActiveModel::Attributes
|
|
51
|
+
|
|
52
|
+
attribute :name, :string
|
|
53
|
+
attribute :age, :integer
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# ... etc
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Then we write benchmarks in files in the `benchmarks/tests` directory. For example:
|
|
60
|
+
|
|
61
|
+
```ruby
|
|
62
|
+
# benchmarks/tests/struct.rb
|
|
63
|
+
|
|
64
|
+
# A group is a collection of related reports
|
|
65
|
+
Awfy.group "Struct" do
|
|
66
|
+
# A report is a collection of tests related to one method or feature we want to benchmark
|
|
67
|
+
report "#some_method" do
|
|
68
|
+
# We do not want to the benchmark to include the creation of the object
|
|
69
|
+
my_struct = MyStruct.new(name: "John", age: 30)
|
|
70
|
+
ruby_struct = Struct.new(:name, :age).new("John", 30)
|
|
71
|
+
dry_struct = DryStruct.new(name: "John", age: 30)
|
|
72
|
+
active_model_attributes = ActiveModelAttributes.new(name: "John", age: 30)
|
|
73
|
+
|
|
74
|
+
# "control" blocks are used to compare the performance to other implementations
|
|
75
|
+
control "Ruby Struct" do
|
|
76
|
+
ruby_struct.some_method
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
control "Dry::Struct" do
|
|
80
|
+
dry_struct.some_method
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
control "ActiveModel::Attributes" do
|
|
84
|
+
active_model_attributes.some_method
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# This is our implementation under test
|
|
88
|
+
test "MyStruct" do
|
|
89
|
+
my_struct.some_method
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
end
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### IPS Benchmarks & Summary Reports
|
|
97
|
+
|
|
98
|
+
Say you are working on performance improvements in a branch called `perf`.
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
git checkout perf
|
|
102
|
+
|
|
103
|
+
# ... make some changes ... then run the benchmarks
|
|
104
|
+
|
|
105
|
+
bundle exec awfy ips Struct "#some_method" --compare-with=main --runtime=both
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Note the comparison here is with the "baseline" which is the "test" block running on MRI without YJIT enabled, on the
|
|
109
|
+
current branch.
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
Running IPS for:
|
|
113
|
+
> Struct/#some_method...
|
|
114
|
+
> [mri - branch 'perf'] Struct / #some_method
|
|
115
|
+
> [mri - branch 'main'] Struct / #some_method
|
|
116
|
+
> [yjit - branch 'perf'] Struct / #some_method
|
|
117
|
+
> [yjit - branch 'main'] Struct / #some_method
|
|
118
|
+
+---------------------------------------------------------------------------+
|
|
119
|
+
| Struct/#some_method |
|
|
120
|
+
+--------+---------+----------------------------+-------------+-------------+
|
|
121
|
+
| Branch | Runtime | Name | IPS | Vs baseline |
|
|
122
|
+
+--------+---------+----------------------------+-------------+-------------+
|
|
123
|
+
| perf | mri | Ruby Struct| 3.288M | 2.26 x |
|
|
124
|
+
| perf | yjit | Ruby Struct| 3.238M | 2.22 x |
|
|
125
|
+
| perf | yjit | MyStruct| 2.364M | 1.62 x |
|
|
126
|
+
| main | yjit | MyStruct| 2.255M | 1.55 x |
|
|
127
|
+
| perf | mri | (baseline) MyStruct| 1.455M | - |
|
|
128
|
+
+--------+---------+----------------------------+-------------+-------------+
|
|
129
|
+
| main | mri | MyStruct| 1.248M | -1.1 x |
|
|
130
|
+
| perf | yjit | Dry::Struct| 1.213M | -1.2 x |
|
|
131
|
+
| perf | mri | Dry::Struct| 639.178k | -2.28 x |
|
|
132
|
+
| perf | yjit | ActiveModel::Attributes| 487.398k | -2.99 x |
|
|
133
|
+
| perf | mri | ActiveModel::Attributes| 310.554k | -4.69 x |
|
|
134
|
+
+--------+---------+----------------------------+-------------+-------------+
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
### Memory Profiling
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
bundle exec awfy memory Struct "#some_method"
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Produces a report like:
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
+----------------------------------------------------------------------------------------------------------------+
|
|
148
|
+
| Struct/.new |
|
|
149
|
+
+--------+---------+----------------------------+-------------------+-------------+----------------+-------------+
|
|
150
|
+
| Branch | Runtime | Name | Total Allocations | Vs baseline | Total Retained | Vs baseline |
|
|
151
|
+
+--------+---------+----------------------------+-------------------+-------------+----------------+-------------+
|
|
152
|
+
| perf | mri | ActiveModel::Attributes | 1.200k | 3.33 x | 640 | ∞ |
|
|
153
|
+
| perf | yjit | ActiveModel::Attributes | 1.200k | 3.33 x | 0 | same |
|
|
154
|
+
| perf | mri | Dry::Struct | 360 | 1.0 x | 160 | ∞ |
|
|
155
|
+
| perf | mri | (baseline) Literal::Struct | 360 | - | 0 | - |
|
|
156
|
+
+--------+---------+----------------------------+-------------------+-------------+----------------+-------------+
|
|
157
|
+
| perf | yjit | Dry::Struct | 360 | same | 0 | same |
|
|
158
|
+
| perf | yjit | Literal::Struct | 360 | same | 0 | same |
|
|
159
|
+
| perf | mri | Ruby Struct | 200 | -0.56 x | 0 | same |
|
|
160
|
+
| perf | mri | Ruby Data | 200 | -0.56 x | 0 | same |
|
|
161
|
+
| perf | yjit | Ruby Struct | 200 | -0.56 x | 0 | same |
|
|
162
|
+
| perf | yjit | Ruby Data | 200 | -0.56 x | 0 | same |
|
|
163
|
+
+--------+---------+----------------------------+-------------------+-------------+----------------+-------------+
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## CLI Options
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
bundle exec awfy -h
|
|
170
|
+
Commands:
|
|
171
|
+
awfy flamegraph GROUP REPORT TEST # Run flamegraph profiling
|
|
172
|
+
awfy help [COMMAND] # Describe available commands or one specific command
|
|
173
|
+
awfy ips [GROUP] [REPORT] [TEST] # Run IPS benchmarks
|
|
174
|
+
awfy list [GROUP] # List all tests in a group
|
|
175
|
+
awfy memory [GROUP] [REPORT] [TEST] # Run memory profiling
|
|
176
|
+
awfy profile [GROUP] [REPORT] [TEST] # Run CPU profiling
|
|
177
|
+
|
|
178
|
+
Options:
|
|
179
|
+
[--runtime=RUNTIME] # Run with and/or without YJIT enabled
|
|
180
|
+
# Default: both
|
|
181
|
+
# Possible values: both, yjit, mri
|
|
182
|
+
[--compare-with=COMPARE_WITH] # Name of branch to compare with results on current branch
|
|
183
|
+
[--compare-control], [--no-compare-control], [--skip-compare-control] # When comparing branches, also re-run all control blocks too
|
|
184
|
+
# Default: false
|
|
185
|
+
[--summary], [--no-summary], [--skip-summary] # Generate a summary of the results
|
|
186
|
+
# Default: true
|
|
187
|
+
[--verbose], [--no-verbose], [--skip-verbose] # Verbose output
|
|
188
|
+
# Default: false
|
|
189
|
+
[--ips-warmup=N] # Number of seconds to warmup the benchmark
|
|
190
|
+
# Default: 1
|
|
191
|
+
[--ips-time=N] # Number of seconds to run the benchmark
|
|
192
|
+
# Default: 3
|
|
193
|
+
[--temp-output-directory=TEMP_OUTPUT_DIRECTORY] # Directory to store temporary output files
|
|
194
|
+
# Default: ./benchmarks/tmp
|
|
195
|
+
[--setup-file-path=SETUP_FILE_PATH] # Path to the setup file
|
|
196
|
+
# Default: ./benchmarks/setup
|
|
197
|
+
[--tests-path=TESTS_PATH] # Path to the tests files
|
|
198
|
+
# Default: ./benchmarks/tests
|
|
199
|
+
```
|
|
26
200
|
|
|
27
201
|
## Development
|
|
28
202
|
|
|
29
203
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
30
204
|
|
|
31
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
32
|
-
|
|
33
205
|
## Contributing
|
|
34
206
|
|
|
35
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
|
207
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/stevegeek/awfy.
|
|
36
208
|
|
|
37
209
|
## License
|
|
38
210
|
|
data/lib/awfy/cli.rb
CHANGED
|
@@ -24,6 +24,7 @@ module Awfy
|
|
|
24
24
|
class_option :compare_control, type: :boolean, desc: "When comparing branches, also re-run all control blocks too", default: false
|
|
25
25
|
|
|
26
26
|
class_option :summary, type: :boolean, desc: "Generate a summary of the results", default: true
|
|
27
|
+
class_option :quiet, type: :boolean, desc: "Silence output. Note if `summary` option is enabled the summaries will be displayed even if `quiet` enabled.", default: false
|
|
27
28
|
class_option :verbose, type: :boolean, desc: "Verbose output", default: false
|
|
28
29
|
|
|
29
30
|
class_option :ips_warmup, type: :numeric, default: 1, desc: "Number of seconds to warmup the benchmark"
|
|
@@ -40,7 +41,7 @@ module Awfy
|
|
|
40
41
|
run_pref_test(group) { list_group(_1) }
|
|
41
42
|
end
|
|
42
43
|
|
|
43
|
-
desc "ips [GROUP] [REPORT] [TEST]", "Run IPS benchmarks"
|
|
44
|
+
desc "ips [GROUP] [REPORT] [TEST]", "Run IPS benchmarks. Can generate summary across implementations, runtimes and branches."
|
|
44
45
|
def ips(group = nil, report = nil, test = nil)
|
|
45
46
|
say "Running IPS for:"
|
|
46
47
|
say "> #{requested_tests(group, report, test)}..."
|
|
@@ -48,7 +49,7 @@ module Awfy
|
|
|
48
49
|
run_pref_test(group) { run_ips(_1, report, test) }
|
|
49
50
|
end
|
|
50
51
|
|
|
51
|
-
desc "memory [GROUP] [REPORT] [TEST]", "Run memory profiling"
|
|
52
|
+
desc "memory [GROUP] [REPORT] [TEST]", "Run memory profiling. Can generate summary across implementations, runtimes and branches."
|
|
52
53
|
def memory(group = nil, report = nil, test = nil)
|
|
53
54
|
say "Running memory profiling for:"
|
|
54
55
|
say "> #{requested_tests(group, report, test)}..."
|
|
@@ -105,6 +106,7 @@ module Awfy
|
|
|
105
106
|
|
|
106
107
|
def run_pref_test(group, &)
|
|
107
108
|
configure_benchmark_run
|
|
109
|
+
prepare_output_directory
|
|
108
110
|
if group
|
|
109
111
|
run_group(group, &)
|
|
110
112
|
else
|
|
@@ -144,12 +146,10 @@ module Awfy
|
|
|
144
146
|
say "> #{group[:name]}...", :cyan
|
|
145
147
|
end
|
|
146
148
|
|
|
147
|
-
prepare_output_directory_for_ips
|
|
148
|
-
|
|
149
149
|
execute_report(group, report_name) do |report, runtime|
|
|
150
150
|
Benchmark.ips(time: options[:ips_time], warmup: options[:ips_warmup], quiet: show_summary? || verbose?) do |bm|
|
|
151
151
|
execute_tests(report, test_name, output: false) do |test, _|
|
|
152
|
-
test_label =
|
|
152
|
+
test_label = generate_test_label(test, runtime)
|
|
153
153
|
bm.item(test_label, &test[:block])
|
|
154
154
|
end
|
|
155
155
|
|
|
@@ -165,18 +165,35 @@ module Awfy
|
|
|
165
165
|
generate_ips_summary if options[:summary]
|
|
166
166
|
end
|
|
167
167
|
|
|
168
|
+
def generate_test_label(test, runtime)
|
|
169
|
+
"[#{runtime}] #{test[:control] ? CONTROL_MARKER : TEST_MARKER} #{test[:name]}"
|
|
170
|
+
end
|
|
171
|
+
|
|
168
172
|
def run_memory(group, report_name, test_name)
|
|
169
173
|
if verbose?
|
|
170
174
|
say "> Memory profiling for:"
|
|
171
175
|
say "> #{group[:name]}...", :cyan
|
|
172
176
|
end
|
|
173
177
|
execute_report(group, report_name) do |report, runtime|
|
|
178
|
+
results = []
|
|
174
179
|
execute_tests(report, test_name) do |test, _|
|
|
175
|
-
MemoryProfiler.report do
|
|
180
|
+
data = MemoryProfiler.report do
|
|
176
181
|
test[:block].call
|
|
177
|
-
end
|
|
182
|
+
end
|
|
183
|
+
test_label = generate_test_label(test, runtime)
|
|
184
|
+
results << {
|
|
185
|
+
label: test_label,
|
|
186
|
+
data: data
|
|
187
|
+
}
|
|
188
|
+
data.pretty_print if verbose?
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
save_to(:memory, group, report, runtime) do |file_name|
|
|
192
|
+
save_memory_profile_report_to_file(file_name, results)
|
|
178
193
|
end
|
|
179
194
|
end
|
|
195
|
+
|
|
196
|
+
generate_memory_summary if options[:summary]
|
|
180
197
|
end
|
|
181
198
|
|
|
182
199
|
def run_flamegraph(group, report_name, test_name)
|
|
@@ -248,7 +265,7 @@ module Awfy
|
|
|
248
265
|
|
|
249
266
|
say if verbose?
|
|
250
267
|
say "> --------------------------" if verbose?
|
|
251
|
-
say ">
|
|
268
|
+
say "> [#{runtime} - branch '#{git_current_branch_name}'] #{group[:name]} / #{report[:name]}", :magenta
|
|
252
269
|
say "> --------------------------" if verbose?
|
|
253
270
|
say if verbose?
|
|
254
271
|
yield run_report, runtime
|
|
@@ -285,11 +302,40 @@ module Awfy
|
|
|
285
302
|
result
|
|
286
303
|
end
|
|
287
304
|
|
|
288
|
-
def
|
|
305
|
+
def prepare_output_directory
|
|
289
306
|
FileUtils.mkdir_p(temp_dir) unless Dir.exist?(temp_dir)
|
|
290
307
|
Dir.glob("#{temp_dir}/*.json").each { |file| File.delete(file) }
|
|
291
308
|
end
|
|
292
309
|
|
|
310
|
+
def save_memory_profile_report_to_file(file_name, results)
|
|
311
|
+
data = results.map do |label_and_data|
|
|
312
|
+
result = label_and_data[:data]
|
|
313
|
+
{
|
|
314
|
+
label: label_and_data[:label],
|
|
315
|
+
total_allocated_memory: result.total_allocated_memsize,
|
|
316
|
+
total_retained_memory: result.total_retained_memsize,
|
|
317
|
+
# Individual results, arrays of objects {count: numeric, data: string}
|
|
318
|
+
allocated_memory_by_gem: result.allocated_memory_by_gem,
|
|
319
|
+
retained_memory_by_gem: result.retained_memory_by_gem,
|
|
320
|
+
allocated_memory_by_file: result.allocated_memory_by_file,
|
|
321
|
+
retained_memory_by_file: result.retained_memory_by_file,
|
|
322
|
+
allocated_memory_by_location: result.allocated_memory_by_location,
|
|
323
|
+
retained_memory_by_location: result.retained_memory_by_location,
|
|
324
|
+
allocated_memory_by_class: result.allocated_memory_by_class,
|
|
325
|
+
retained_memory_by_class: result.retained_memory_by_class,
|
|
326
|
+
allocated_objects_by_gem: result.allocated_objects_by_gem,
|
|
327
|
+
retained_objects_by_gem: result.retained_objects_by_gem,
|
|
328
|
+
allocated_objects_by_file: result.allocated_objects_by_file,
|
|
329
|
+
retained_objects_by_file: result.retained_objects_by_file,
|
|
330
|
+
allocated_objects_by_location: result.allocated_objects_by_location,
|
|
331
|
+
retained_objects_by_location: result.retained_objects_by_location,
|
|
332
|
+
allocated_objects_by_class: result.allocated_objects_by_class,
|
|
333
|
+
retained_objects_by_class: result.retained_objects_by_class
|
|
334
|
+
}
|
|
335
|
+
end
|
|
336
|
+
File.write(file_name, data.to_json)
|
|
337
|
+
end
|
|
338
|
+
|
|
293
339
|
def save_to(type, group, report, runtime)
|
|
294
340
|
current_branch = git_current_branch_name
|
|
295
341
|
file_name = "#{temp_dir}/#{type}-#{runtime}-#{current_branch}-#{group[:name]}-#{report[:name]}.json".gsub(/[^A-Za-z0-9\/_\-.]/, "_")
|
|
@@ -304,7 +350,18 @@ module Awfy
|
|
|
304
350
|
yield file_name
|
|
305
351
|
end
|
|
306
352
|
|
|
307
|
-
def
|
|
353
|
+
def load_results_json(type, file_name)
|
|
354
|
+
case type
|
|
355
|
+
when "ips"
|
|
356
|
+
load_ips_results_json(file_name)
|
|
357
|
+
when "memory"
|
|
358
|
+
load_memory_results_json(file_name)
|
|
359
|
+
else
|
|
360
|
+
raise "Unknown test type"
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def load_ips_results_json(file_name)
|
|
308
365
|
JSON.parse(File.read(file_name)).map do |result|
|
|
309
366
|
{
|
|
310
367
|
label: result["item"],
|
|
@@ -316,14 +373,121 @@ module Awfy
|
|
|
316
373
|
end
|
|
317
374
|
end
|
|
318
375
|
|
|
319
|
-
def
|
|
320
|
-
|
|
376
|
+
def load_memory_results_json(file_name)
|
|
377
|
+
JSON.parse(File.read(file_name)).map { _1.transform_keys(&:to_sym) }
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
def choose_baseline_test(results)
|
|
381
|
+
base_branch = git_current_branch_name
|
|
382
|
+
baseline = results.find do |r|
|
|
383
|
+
r[:branch] == base_branch && r[:label].include?(TEST_MARKER) && r[:runtime] == (yjit_only? ? "yjit" : "mri") # Baseline is mri baseline unless yjit only
|
|
384
|
+
end
|
|
385
|
+
unless baseline
|
|
386
|
+
say_error "Could not work out which result is considered the 'baseline' (ie the `test` case)"
|
|
387
|
+
exit(1)
|
|
388
|
+
end
|
|
389
|
+
baseline[:is_baseline] = true
|
|
390
|
+
say "> Chosen baseline: #{baseline[:label]}" if verbose?
|
|
391
|
+
baseline
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def generate_memory_summary
|
|
395
|
+
read_reports_for_summary("memory") do |report, results, baseline|
|
|
396
|
+
result_diffs = results.map do |result|
|
|
397
|
+
overlaps = result[:total_allocated_memory] == baseline[:total_allocated_memory] && result[:total_retained_memory] == baseline[:total_retained_memory]
|
|
398
|
+
diff_x = if baseline[:total_allocated_memory].zero? && !result[:total_allocated_memory].zero?
|
|
399
|
+
Float::INFINITY
|
|
400
|
+
elsif baseline[:total_allocated_memory].zero?
|
|
401
|
+
0.0
|
|
402
|
+
elsif baseline[:total_allocated_memory] > result[:total_allocated_memory]
|
|
403
|
+
-1.0 * result[:total_allocated_memory] / baseline[:total_allocated_memory]
|
|
404
|
+
else
|
|
405
|
+
result[:total_allocated_memory].to_f / baseline[:total_allocated_memory]
|
|
406
|
+
end
|
|
407
|
+
retained_diff_x = if baseline[:total_retained_memory].zero? && !result[:total_retained_memory].zero?
|
|
408
|
+
Float::INFINITY
|
|
409
|
+
elsif baseline[:total_retained_memory].zero?
|
|
410
|
+
0.0
|
|
411
|
+
elsif baseline[:total_retained_memory] > result[:total_retained_memory]
|
|
412
|
+
-1.0 * result[:total_retained_memory] / baseline[:total_retained_memory]
|
|
413
|
+
else
|
|
414
|
+
result[:total_retained_memory].to_f / baseline[:total_retained_memory]
|
|
415
|
+
end
|
|
416
|
+
result.merge(
|
|
417
|
+
overlaps: overlaps,
|
|
418
|
+
diff_times: diff_x.round(2),
|
|
419
|
+
retained_diff_times: retained_diff_x.round(2)
|
|
420
|
+
)
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
result_diffs.sort_by! { |result| -1 * result[:diff_times] }
|
|
424
|
+
|
|
425
|
+
rows = result_diffs.map do |result|
|
|
426
|
+
diff_message = result_diff_message(result)
|
|
427
|
+
retained_message = result_diff_message(result, :retained_diff_times)
|
|
428
|
+
test_name = result[:is_baseline] ? "(baseline) #{result[:test_name]}" : result[:test_name]
|
|
429
|
+
[result[:branch], result[:runtime], test_name, humanize_scale(result[:total_allocated_memory]), diff_message, humanize_scale(result[:total_retained_memory]), retained_message]
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
output_summary_table(report, rows, "Branch", "Runtime", "Name", "Total Allocations", "Vs baseline", "Total Retained", "Vs baseline")
|
|
433
|
+
end
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def output_summary_table(report, rows, *headings)
|
|
437
|
+
group_data = report.first
|
|
438
|
+
table = ::Terminal::Table.new(title: requested_tests(group_data[:group], group_data[:report]), headings: headings)
|
|
439
|
+
|
|
440
|
+
rows.each do |row|
|
|
441
|
+
table.add_row(row)
|
|
442
|
+
if row[4] == "-" # FIXME: this is finding the baseline...
|
|
443
|
+
table.add_separator(border_type: :dot3)
|
|
444
|
+
end
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
(2...headings.size).each { table.align_column(_1, :right) }
|
|
448
|
+
|
|
449
|
+
if options[:quiet] && options[:summary]
|
|
450
|
+
puts table
|
|
451
|
+
else
|
|
452
|
+
say table
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def result_diff_message(result, diff_key = :diff_times)
|
|
457
|
+
if result[:is_baseline]
|
|
458
|
+
"-"
|
|
459
|
+
elsif result[:overlaps] || result[diff_key].zero?
|
|
460
|
+
"same"
|
|
461
|
+
elsif result[diff_key] == Float::INFINITY
|
|
462
|
+
"∞"
|
|
463
|
+
elsif result[diff_key]
|
|
464
|
+
"#{result[diff_key]} x"
|
|
465
|
+
else
|
|
466
|
+
"?"
|
|
467
|
+
end
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
SUFFIXES = ["", "k", "M", "B", "T", "Q"].freeze
|
|
471
|
+
|
|
472
|
+
def humanize_scale(number, round_to: 0)
|
|
473
|
+
return 0 if number.zero?
|
|
474
|
+
number = number.round(round_to)
|
|
475
|
+
scale = (Math.log10(number) / 3).to_i
|
|
476
|
+
scale = 0 if scale < 0 || scale >= SUFFIXES.size
|
|
477
|
+
suffix = SUFFIXES[scale]
|
|
478
|
+
scaled_value = number.to_f / (1000**scale)
|
|
479
|
+
dp = (scale == 0) ? 0 : 3
|
|
480
|
+
"%10.#{dp}f#{suffix}" % scaled_value
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def read_reports_for_summary(type)
|
|
484
|
+
awfy_report_result_files = Dir.glob("#{temp_dir}/awfy-#{type}-*.json").map do |file_name|
|
|
321
485
|
JSON.parse(File.read(file_name)).map { _1.transform_keys(&:to_sym) }
|
|
322
486
|
end
|
|
323
487
|
|
|
324
488
|
awfy_report_result_files.each do |report|
|
|
325
489
|
results = report.map do |single_run|
|
|
326
|
-
|
|
490
|
+
load_results_json(type, single_run[:output_path]).map do |result|
|
|
327
491
|
test_name = result[:label].match(/\[.{3,4}\] \[.\] (.*)/)[1]
|
|
328
492
|
result.merge!(
|
|
329
493
|
runtime: single_run[:runtime],
|
|
@@ -333,65 +497,39 @@ module Awfy
|
|
|
333
497
|
end
|
|
334
498
|
end
|
|
335
499
|
results.flatten!(1)
|
|
500
|
+
baseline = choose_baseline_test(results)
|
|
336
501
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
end
|
|
341
|
-
unless baseline
|
|
342
|
-
say_error "Could not work out which result is considered the 'baseline' (ie the `test` case)"
|
|
343
|
-
exit(1)
|
|
344
|
-
end
|
|
345
|
-
baseline[:is_baseline] = true
|
|
346
|
-
say "> Chosen baseline: #{baseline[:label]}" if verbose?
|
|
502
|
+
yield report, results, baseline
|
|
503
|
+
end
|
|
504
|
+
end
|
|
347
505
|
|
|
506
|
+
def generate_ips_summary
|
|
507
|
+
read_reports_for_summary("ips") do |report, results, baseline|
|
|
348
508
|
result_diffs = results.map do |result|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
result_stats.slowdown(baseline_stats).first
|
|
357
|
-
end
|
|
509
|
+
baseline_stats = baseline[:stats]
|
|
510
|
+
result_stats = result[:stats]
|
|
511
|
+
overlaps = result_stats.overlaps?(baseline_stats)
|
|
512
|
+
diff_x = if baseline_stats.central_tendency > result_stats.central_tendency
|
|
513
|
+
-1.0 * result_stats.speedup(baseline_stats).first
|
|
514
|
+
else
|
|
515
|
+
result_stats.slowdown(baseline_stats).first
|
|
358
516
|
end
|
|
359
|
-
|
|
360
517
|
result.merge(
|
|
361
518
|
overlaps: overlaps,
|
|
362
|
-
diff_times: diff_x
|
|
519
|
+
diff_times: diff_x.round(2)
|
|
363
520
|
)
|
|
364
521
|
end
|
|
365
522
|
|
|
366
523
|
result_diffs.sort_by! { |result| -1 * result[:iter] }
|
|
367
524
|
|
|
368
525
|
rows = result_diffs.map do |result|
|
|
369
|
-
diff_message =
|
|
370
|
-
|
|
371
|
-
elsif result[:overlaps]
|
|
372
|
-
"same-ish"
|
|
373
|
-
elsif result[:diff_times]
|
|
374
|
-
"#{result[:diff_times].round(2)} x"
|
|
375
|
-
else
|
|
376
|
-
"?"
|
|
377
|
-
end
|
|
378
|
-
test_name = result[:is_baseline] ? "#{result[:test_name]} (baseline)" : result[:test_name]
|
|
526
|
+
diff_message = result_diff_message(result)
|
|
527
|
+
test_name = result[:is_baseline] ? "(baseline) #{result[:test_name]}" : result[:test_name]
|
|
379
528
|
|
|
380
|
-
[result[:branch], result[:runtime], test_name, result[:stats].central_tendency
|
|
529
|
+
[result[:branch], result[:runtime], test_name, humanize_scale(result[:stats].central_tendency), diff_message]
|
|
381
530
|
end
|
|
382
531
|
|
|
383
|
-
|
|
384
|
-
table = ::Terminal::Table.new(
|
|
385
|
-
title: "Summary for #{requested_tests(group_data[:group], group_data[:report])}",
|
|
386
|
-
headings: ["Branch", "Runtime", "Name", "IPS", "Diff v baseline (times)"],
|
|
387
|
-
rows: rows
|
|
388
|
-
)
|
|
389
|
-
|
|
390
|
-
table.align_column(2, :right)
|
|
391
|
-
table.align_column(3, :right)
|
|
392
|
-
table.align_column(4, :right)
|
|
393
|
-
|
|
394
|
-
say table
|
|
532
|
+
output_summary_table(report, rows, "Branch", "Runtime", "Name", "IPS", "Vs baseline")
|
|
395
533
|
end
|
|
396
534
|
end
|
|
397
535
|
|
data/lib/awfy/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: awfy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Stephen Ierodiaconou
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-10-
|
|
11
|
+
date: 2024-10-29 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|