awfy 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bea684a36c13b8593199aa4490e13c5fbdcb35222766daa401b9c8151a71c8f
-  data.tar.gz: 1825c25d81bd2210269d713313a8634d66f654b13e763e1f4bb33f7f62bcfe67
+  metadata.gz: 9b3e0b8df98fcef36e265d46ab9416c1b7cd529116fe57a4b9470a95d04427d0
+  data.tar.gz: a7d9962369da0320d5d97cfb32db82ff06b52d72425e22e4134759c8a549a140
 SHA512:
-  metadata.gz: 273fda35ea1688650b64b815d7b8b37b15485214341de4e2ca7e048a3f84f89cc0cb5693cadd5bfb0ed14810eb2169f142080e82df26c60acdbc1b421e046403
-  data.tar.gz: d00c1252bbb61fe0687806be84dcc6cdc2ce20148b893fe50f09dc82fa258ec0f6f4deb1a89aefae0efcb51c187aa3a2928db748dfa321f096b4f71b328e0465
+  metadata.gz: 89e5d7f45b6375e790ea7387af082d603d3464caa0944084970c858c526e4bccb95673a4beddb61ebfb0b3b3bb64b7b285b613870b41d1baa3ed7d114d9c9ee8
+  data.tar.gz: beed2c0d694ffb1da281c9ac0344ca52e11cb981c2933d34f6a26ef5b73ba16c4d2e525bd40dd9a9fe68fc2a51bf10a3870253330eaa540aa05936ef7d211bb6

data/README.md

@@ -1,38 +1,181 @@
 # Awfy (Are We Fast Yet)
 
-TODO: Delete this and the text below, and describe your gem
+CLI tool to help run suites of benchmarks and compare results between control implementations, across branches and with or without YJIT.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/awfy`. To experiment with that code, run `bin/console` for an interactive prompt.
+The benchmarks are written using a simple DSL in your target project.
 
-## Installation
+Supports running:
+
+- IPS benchmarks (with [benchmark-ips](https://rubygems.org/gems/benchmark-ips))
+- Memory profiling (with [memory_profiler](https://rubygems.org/gems/memory_profiler))
+- CPU profiling (with [stackprof](https://rubygems.org/gems/stackprof))
+- Flamegraph profiling (with [singed](https://rubygems.org/gems/singed))
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+Awfy can also create summary reports of the results which can be useful for comparing the performance of different implementations **(currently only supported for IPS benchmarks)**.
+
+## Installation
 
 Install the gem and add to the application's Gemfile by executing:
 
 ```bash
-bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+bundle add awfy
 ```
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
 ```bash
-gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+gem install awfy
 ```
 
 ## Usage
 
-TODO: Write usage instructions here
+Imagine we have a custom implementation of a Struct class called `MyStruct`. We want to compare the performance of our implementation with the built-in `Struct` class
+and other similar implementations.
+
+First, we need to create a setup file in the `benchmarks/setup.rb` directory. For example:
+
+```ruby
+# setup.rb
+
+require "dry-struct"
+require "active_model"
+
+class DryStruct < Dry::Struct
+  attribute :name, Types::String
+  attribute :age, Types::Integer
+end
+
+class ActiveModelAttributes
+  include ActiveModel::API
+  include ActiveModel::Attributes
+  
+  attribute :name, :string
+  attribute :age, :integer
+end
+
+# ... etc
+```
+
+Then we write benchmarks in files in the `benchmarks/tests` directory. For example:
+
+```ruby
+# benchmarks/tests/struct.rb
+
+# A group is a collection of related reports
+Awfy.group "Struct" do
+  # A report is a collection of tests related to one method or feature we want to benchmark
+  report "#some_method" do
+    # We do not want to the benchmark to include the creation of the object
+    my_struct = MyStruct.new(name: "John", age: 30)
+    ruby_struct = Struct.new(:name, :age).new("John", 30)
+    dry_struct = DryStruct.new(name: "John", age: 30)
+    active_model_attributes = ActiveModelAttributes.new(name: "John", age: 30)
+    
+    # "control" blocks are used to compare the performance to other implementations
+    control "Ruby Struct" do
+      ruby_struct.some_method
+    end
+    
+    control "Dry::Struct" do
+      dry_struct.some_method
+    end
+    
+    control "ActiveModel::Attributes" do
+      active_model_attributes.some_method
+    end
+    
+    # This is our implementation under test
+    test "MyStruct" do
+      my_struct.some_method
+    end  
+  end
+  
+end
+```
+
+### IPS Benchmarks & Summary Reports
+
+Say you are working on performance improvements in a branch called `perf`.
+
+```bash
+git checkout perf
+
+# ... make some changes ... then run the benchmarks
+
+bundle exec awfy ips Struct "#some_method" --compare-with=main --runtime=both
+```
+
+Note the comparison here is with the "baseline" which is the "test" block running on MRI without YJIT enabled, on the
+current branch.
+
+```
+Running IPS for:
+> Struct/#some_method...
+> [mri - branch 'perf'] Struct / #some_method
+> [mri - branch 'main'] Struct / #some_method
+> [yjit - branch 'perf'] Struct / #some_method
+> [yjit - branch 'main'] Struct / #some_method
++---------------------------------------------------------------------------+
+|                           Struct/#some_method                             |
++--------+---------+----------------------------+-------------+-------------+
+| Branch | Runtime | Name                       | IPS         | Vs baseline |
++--------+---------+----------------------------+-------------+-------------+
+| perf   | mri     | Ruby Struct                |      3.288M | 2.26 x      |
+| perf   | yjit    | Ruby Struct                |      3.238M | 2.22 x      |
+| perf   | yjit    | MyStruct                   |      2.364M | 1.62 x      |
+| main   | yjit    | MyStruct                   |      2.255M | 1.55 x      |
+| perf   | mri     | (baseline) MyStruct        |      1.455M | -           |
++--------+---------+----------------------------+-------------+-------------+
+| main   | mri     | MyStruct                   |      1.248M | -1.1 x      |
+| perf   | yjit    | Dry::Struct                |      1.213M | -1.2 x      |
+| perf   | mri     | Dry::Struct                |    639.178k | -2.28 x     |
+| perf   | yjit    | ActiveModel::Attributes    |    487.398k | -2.99 x     |
+| perf   | mri     | ActiveModel::Attributes    |    310.554k | -4.69 x     |
++--------+---------+----------------------------+-------------+-------------+
+```
+
+## CLI Options
+
+```
+bundle exec awfy -h
+Commands:
+  awfy flamegraph GROUP REPORT TEST     # Run flamegraph profiling
+  awfy help [COMMAND]                   # Describe available commands or one specific command
+  awfy ips [GROUP] [REPORT] [TEST]      # Run IPS benchmarks
+  awfy list [GROUP]                     # List all tests in a group
+  awfy memory [GROUP] [REPORT] [TEST]   # Run memory profiling
+  awfy profile [GROUP] [REPORT] [TEST]  # Run CPU profiling
+
+Options:
+  [--runtime=RUNTIME]                                                    # Run with and/or without YJIT enabled
+                                                                         # Default: both
+                                                                         # Possible values: both, yjit, mri
+  [--compare-with=COMPARE_WITH]                                          # Name of branch to compare with results on current branch
+  [--compare-control], [--no-compare-control], [--skip-compare-control]  # When comparing branches, also re-run all control blocks too
+                                                                         # Default: false
+  [--summary], [--no-summary], [--skip-summary]                          # Generate a summary of the results
+                                                                         # Default: true
+  [--verbose], [--no-verbose], [--skip-verbose]                          # Verbose output
+                                                                         # Default: false
+  [--ips-warmup=N]                                                       # Number of seconds to warmup the benchmark
+                                                                         # Default: 1
+  [--ips-time=N]                                                         # Number of seconds to run the benchmark
+                                                                         # Default: 3
+  [--temp-output-directory=TEMP_OUTPUT_DIRECTORY]                        # Directory to store temporary output files
+                                                                         # Default: ./benchmarks/tmp
+  [--setup-file-path=SETUP_FILE_PATH]                                    # Path to the setup file
+                                                                         # Default: ./benchmarks/setup
+  [--tests-path=TESTS_PATH]                                              # Path to the tests files
+                                                                         # Default: ./benchmarks/tests
+```
 
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/awfy.
+Bug reports and pull requests are welcome on GitHub at https://github.com/stevegeek/awfy.
 
 ## License
 

data/lib/awfy/cli.rb

@@ -25,7 +25,6 @@ module Awfy
 
     class_option :summary, type: :boolean, desc: "Generate a summary of the results", default: true
     class_option :verbose, type: :boolean, desc: "Verbose output", default: false
-    class_option :quiet, type: :boolean, desc: "Silence output", default: false
 
     class_option :ips_warmup, type: :numeric, default: 1, desc: "Number of seconds to warmup the benchmark"
     class_option :ips_time, type: :numeric, default: 3, desc: "Number of seconds to run the benchmark"
@@ -48,33 +47,34 @@ module Awfy
 
       run_pref_test(group) { run_ips(_1, report, test) }
     end
-    #
-    # desc "memory [GROUP] [REPORT] [TEST]", "Run memory profiling"
-    # def memory(group = nil, report = nil, test = nil)
-    #   say "Running memory profiling for:"
-    #   say "> #{requested_tests(group, report, test)}..."
-    #
-    #   run_pref_test(group) { run_memory(_1, report, test) }
-    # end
-    #
-    # desc "flamegraph GROUP REPORT TEST", "Run flamegraph profiling"
-    # def flamegraph(group, report, test)
-    #   say "Creating flamegraph for:"
-    #   say "> #{[group, report, test].join("/")}..."
-    #   configure_benchmark_run
-    #   run_group(group) { run_flamegraph(_1, report, test) }
-    # end
-    #
+
+    desc "memory [GROUP] [REPORT] [TEST]", "Run memory profiling"
+    def memory(group = nil, report = nil, test = nil)
+      say "Running memory profiling for:"
+      say "> #{requested_tests(group, report, test)}..."
+
+      run_pref_test(group) { run_memory(_1, report, test) }
+    end
+
+    desc "flamegraph GROUP REPORT TEST", "Run flamegraph profiling"
+    def flamegraph(group, report, test)
+      say "Creating flamegraph for:"
+      say "> #{[group, report, test].join("/")}..."
+
+      configure_benchmark_run
+      run_group(group) { run_flamegraph(_1, report, test) }
+    end
+
     # # TODO: also YJIT stats output?
-    # desc "profile [GROUP] [REPORT] [TEST]", "Run CPU profiling"
-    # option :iterations, type: :numeric, default: 1_000_000, desc: "Number of iterations to run the test"
-    # def profile(group = nil, report = nil, test = nil)
-    #   say "Run profiling of:"
-    #   say "> #{requested_tests(group, report, test)}..."
-    #
-    #   configure_benchmark_run
-    #   run_group(group) { run_profiling(_1, report, test) }
-    # end
+    desc "profile [GROUP] [REPORT] [TEST]", "Run CPU profiling"
+    option :iterations, type: :numeric, default: 1_000_000, desc: "Number of iterations to run the test"
+    def profile(group = nil, report = nil, test = nil)
+      say "Run profiling of:"
+      say "> #{requested_tests(group, report, test)}..."
+
+      configure_benchmark_run
+      run_group(group) { run_profiling(_1, report, test) }
+    end
 
     private
 
@@ -147,7 +147,7 @@ module Awfy
       prepare_output_directory_for_ips
 
       execute_report(group, report_name) do |report, runtime|
-        Benchmark.ips(time: options[:ips_time], warmup: options[:ips_warmup], quiet: quiet_steps?) do |bm|
+        Benchmark.ips(time: options[:ips_time], warmup: options[:ips_warmup], quiet: show_summary? || verbose?) do |bm|
           execute_tests(report, test_name, output: false) do |test, _|
             test_label = "[#{runtime}] #{test[:control] ? CONTROL_MARKER : TEST_MARKER} #{test[:name]}"
             bm.item(test_label, &test[:block])
@@ -158,50 +158,56 @@ module Awfy
             bm.save!(file_name)
           end
 
-          bm.compare! if verbose?
+          bm.compare! if verbose? || !show_summary?
         end
       end
 
       generate_ips_summary if options[:summary]
     end
-    #
-    # def run_memory(group, report_name, test_name)
-    #   say "> Memory profiling for #{group[:name]}...", :cyan if verbose?
-    #   execute_report(group, report_name) do |report, runtime|
-    #     execute_tests(report, test_name) do |test, _|
-    #       MemoryProfiler.report do
-    #         test[:block].call
-    #       end.pretty_print
-    #     end
-    #   end
-    # end
-    #
-    # def run_flamegraph(group, report_name, test_name)
-    #   execute_report(group, report_name)  do |report, runtime|
-    #     execute_tests(report, test_name) do |test, _|
-    #       label = "report-#{group[:name]}-#{report[:name]}-#{test[:name]}".gsub(/[^A-Za-z0-9_\-]/, "_")
-    #       generate_flamegraph(label) do
-    #         test[:block].call
-    #       end
-    #     end
-    #   end
-    # end
-    #
-    # def run_profiling(group, report_name, test_name)
-    #   say "> Profiling for #{group[:name]} (iterations: #{options[:iterations]})..." if verbose?
-    #   execute_report(group, report_name) do |report, runtime|
-    #     execute_tests(report, test_name) do |test, iterations|
-    #       data = StackProf.run(mode: :cpu, interval: 100) do
-    #         i = 0
-    #         while i < iterations
-    #           test[:block].call
-    #           i += 1
-    #         end
-    #       end
-    #       StackProf::Report.new(data).print_text
-    #     end
-    #   end
-    # end
+
+    def run_memory(group, report_name, test_name)
+      if verbose?
+        say "> Memory profiling for:"
+        say "> #{group[:name]}...", :cyan
+      end
+      execute_report(group, report_name) do |report, runtime|
+        execute_tests(report, test_name) do |test, _|
+          MemoryProfiler.report do
+            test[:block].call
+          end.pretty_print
+        end
+      end
+    end
+
+    def run_flamegraph(group, report_name, test_name)
+      execute_report(group, report_name) do |report, runtime|
+        execute_tests(report, test_name) do |test, _|
+          label = "report-#{group[:name]}-#{report[:name]}-#{test[:name]}".gsub(/[^A-Za-z0-9_\-]/, "_")
+          generate_flamegraph(label) do
+            test[:block].call
+          end
+        end
+      end
+    end
+
+    def run_profiling(group, report_name, test_name)
+      if verbose?
+        say "> Profiling for:"
+        say "> #{group[:name]} (iterations: #{options[:iterations]})...", :cyan
+      end
+      execute_report(group, report_name) do |report, runtime|
+        execute_tests(report, test_name) do |test, iterations|
+          data = StackProf.run(mode: :cpu, interval: 100) do
+            i = 0
+            while i < iterations
+              test[:block].call
+              i += 1
+            end
+          end
+          StackProf::Report.new(data).print_text
+        end
+      end
+    end
 
     def execute_report(group, report_name, &)
       runtime = options[:runtime]
@@ -242,7 +248,7 @@ module Awfy
 
         say if verbose?
         say "> --------------------------" if verbose?
-        say "> Report (#{runtime} - branch '#{git_current_branch_name}'): #{report[:name]}"
+        say "> [#{runtime} - branch '#{git_current_branch_name}'] #{group[:name]} / #{report[:name]}", :magenta
         say "> --------------------------" if verbose?
         say if verbose?
         yield run_report, runtime
@@ -369,22 +375,28 @@ module Awfy
           else
             "?"
           end
-          test_name = result[:is_baseline] ? "#{result[:test_name]} (baseline)" : result[:test_name]
+          test_name = result[:is_baseline] ? "(baseline) #{result[:test_name]}" : result[:test_name]
 
-          [result[:branch], result[:runtime], test_name, result[:stats].central_tendency.round, diff_message]
+          [result[:branch], result[:runtime], test_name, Benchmark::IPS::Helpers.scale(result[:stats].central_tendency.round), diff_message]
         end
 
         group_data = report.first
         table = ::Terminal::Table.new(
-          title: "Summary for #{requested_tests(group_data[:group], group_data[:report])}",
-          headings: ["Branch", "Runtime", "Name", "IPS", "Diff v baseline (times)"],
-          rows: rows
+          title: requested_tests(group_data[:group], group_data[:report]),
+          headings: ["Branch", "Runtime", "Name", "IPS", "Vs baseline"]
         )
 
         table.align_column(2, :right)
         table.align_column(3, :right)
         table.align_column(4, :right)
 
+        rows.each do |row|
+          table.add_row(row)
+          if row[4] == "-"
+            table.add_separator(border_type: :dot3)
+          end
+        end
+
         say table
       end
     end
@@ -403,7 +415,8 @@ module Awfy
     ensure
       say "Switching back to branch '#{previous_branch}'" if verbose?
       git_client.checkout(previous_branch)
-      git_client.lib.stash_apply(0)
+      # Git client does not have a pop method so send our own command
+      git_client.lib.send(:command, "stash", "pop")
     end
 
     def git_current_branch_name = git_client.current_branch
@@ -416,6 +429,6 @@ module Awfy
 
     def verbose? = options[:verbose]
 
-    def quiet_steps? = options[:quiet] || options[:summary]
+    def show_summary? = options[:summary]
   end
 end

data/lib/awfy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Awfy
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: awfy
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Stephen Ierodiaconou
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-28 00:00:00.000000000 Z
+date: 2024-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor