derailed_benchmarks 1.3.6 → 1.4.0

data/lib/derailed_benchmarks/stats_in_file.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module DerailedBenchmarks
+  # A class for reading in benchmark results
+  # and converting them to numbers for comparison
+  #
+  # Example:
+  #
+  #  puts `cat muhfile.bench.txt`
+  #
+  #    9.590142   0.831269  10.457801 ( 10.0)
+  #    9.836019   0.837319  10.728024 ( 11.0)
+  #
+  #  x = StatsForFile.new(name: "muhcommit", file: "muhfile.bench.txt", desc: "I made it faster", time: Time.now)
+  #  x.values  #=> [11.437769, 11.792425]
+  #  x.average # => 10.5
+  #  x.name    # => "muhfile"
+  class StatsForFile
+    attr_reader :name, :values, :desc, :time
+
+    def initialize(file:, name:, desc: "", time: )
+      @file = Pathname.new(file)
+      FileUtils.touch(@file)
+
+      @name = name
+      @desc = desc
+      @time = time
+    end
+
+    def call
+      load_file!
+
+      @average = values.inject(:+) / values.length
+    end
+
+    def average
+      @average.to_f
+    end
+
+    private def load_file!
+      @values = []
+      @file.each_line do |line|
+        line.match(/\( +(\d+\.\d+)\)/)
+        begin
+          values << BigDecimal($1)
+        rescue => e
+          raise e, "Problem with file #{@file.inspect}:\n#{@file.read}\n#{e.message}"
+        end
+      end
+      values.freeze
+    end
+  end
+end

data/lib/derailed_benchmarks/tasks.rb

@@ -1,137 +1,81 @@
-namespace :perf do
-  task :rails_load do
-    ENV["RAILS_ENV"] ||= "production"
-    ENV['RACK_ENV']  = ENV["RAILS_ENV"]
-    ENV["DISABLE_SPRING"] = "true"
-
-    ENV["SECRET_KEY_BASE"] ||= "foofoofoo"
-
-    ENV['LOG_LEVEL'] ||= "FATAL"
-
-    require 'rails'
-
-    puts "Booting: #{Rails.env}"
+require_relative 'load_tasks'
 
-    %W{ . lib test config }.each do |file|
-      $LOAD_PATH << File.expand_path(file)
-    end
-
-    require 'application'
-
-    Rails.env = ENV["RAILS_ENV"]
+namespace :perf do
+  desc "runs the same test against two different branches for statistical comparison"
+  task :library do
+    DERAILED_SCRIPT_COUNT = (ENV["DERAILED_SCRIPT_COUNT"] ||= "200").to_i
+    ENV["TEST_COUNT"] ||= "200"
 
-    DERAILED_APP = Rails.application
+    raise "test count must be at least 2, is set to #{DERAILED_SCRIPT_COUNT}" if DERAILED_SCRIPT_COUNT < 2
+    script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
 
-    if DERAILED_APP.respond_to?(:initialized?)
-      DERAILED_APP.initialize! unless DERAILED_APP.initialized?
+    if ENV["DERAILED_PATH_TO_LIBRARY"]
+      library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
     else
-      DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
+      library_dir = DerailedBenchmarks.rails_path_on_disk
     end
 
-    if  ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
-      if defined? ActiveRecord::Tasks::DatabaseTasks
-        ActiveRecord::Tasks::DatabaseTasks.create_current
-      else # Rails 3.2
-        raise "No valid database for #{ENV['RAILS_ENV']}, please create one" unless ActiveRecord::Base.connection.active?.inspect
+    raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
+
+    # Use either the explicit SHAs when present or grab last two SHAs from commit history
+    # if only one SHA is given, then use it and the last SHA from commit history
+    branch_names = []
+    branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
+    if branch_names.length < 2
+      Dir.chdir(library_dir) do
+        branches = run!('git log --format="%H" -n 2').chomp.split($INPUT_RECORD_SEPARATOR)
+        if branch_names.empty?
+          branch_names = branches
+        else
+          branch_names << branches.shift
+        end
       end
-
-      ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
-      ActiveRecord::Migration.verbose         = true
-      ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
     end
 
-    DERAILED_APP.config.consider_all_requests_local = true
-  end
+    current_library_branch = ""
+    Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
 
-  task :rack_load do
-    puts "You're not using Rails"
-    puts "You need to tell derailed how to boot your app"
-    puts "In your perf.rake add:"
-    puts
-    puts "namespace :perf do"
-    puts "  task :rack_load do"
-    puts "    # DERAILED_APP = your code here"
-    puts "  end"
-    puts "end"
-  end
+    out_dir = Pathname.new("tmp/library_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
+    out_dir.mkpath
 
-  task :setup do
-    if DerailedBenchmarks.gem_is_bundled?("railties")
-      Rake::Task["perf:rails_load"].invoke
-    else
-      Rake::Task["perf:rack_load"].invoke
-    end
-
-    WARM_COUNT  = (ENV['WARM_COUNT'] || 0).to_i
-    TEST_COUNT  = (ENV['TEST_COUNT'] || ENV['CNT'] || 1_000).to_i
-    PATH_TO_HIT = ENV["PATH_TO_HIT"] || ENV['ENDPOINT'] || "/"
-    puts "Endpoint: #{ PATH_TO_HIT.inspect }"
+    branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
+    branch_info = {}
 
-    HTTP_HEADER_PREFIX = "HTTP_".freeze
-    RACK_HTTP_HEADERS = ENV.select { |key| key.start_with?(HTTP_HEADER_PREFIX) }
-
-    HTTP_HEADERS = RACK_HTTP_HEADERS.keys.inject({}) do |hash, rack_header_name|
-      # e.g. "HTTP_ACCEPT_CHARSET" -> "Accept-Charset"
-      header_name = rack_header_name[HTTP_HEADER_PREFIX.size..-1].split("_").map(&:downcase).map(&:capitalize).join("-")
-      hash[header_name] = RACK_HTTP_HEADERS[rack_header_name]
-      hash
-    end
-    puts "HTTP headers: #{HTTP_HEADERS}" unless HTTP_HEADERS.empty?
-
-    CURL_HTTP_HEADER_ARGS = HTTP_HEADERS.map { |http_header_name, value| "-H \"#{http_header_name}: #{value}\"" }.join(" ")
-
-    require 'rack/test'
-    require 'rack/file'
-
-    DERAILED_APP = DerailedBenchmarks.add_auth(Object.class_eval { remove_const(:DERAILED_APP) })
-    if server = ENV["USE_SERVER"]
-      @port = (3000..3900).to_a.sample
-      puts "Port: #{ @port.inspect }"
-      puts "Server: #{ server.inspect }"
-      thread = Thread.new do
-        Rack::Server.start(app: DERAILED_APP, :Port => @port, environment: "none", server: server)
+    branches_to_test.each do |branch, file|
+      Dir.chdir(library_dir) do
+        run!("git checkout '#{branch}'")
+        description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
+        time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
+        name        = run!("git rev-parse --short HEAD").strip
+        branch_info[name] = { desc: description, time: DateTime.parse(time_stamp), file: file }
       end
-      sleep 1
-
-      def call_app(path = File.join("/", PATH_TO_HIT))
-        cmd = "curl #{CURL_HTTP_HEADER_ARGS} 'http://localhost:#{@port}#{path}' -s --fail 2>&1"
-        response = `#{cmd}`
-        unless $?.success?
-          STDERR.puts "Couldn't call app."
-          STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
-
-          FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+      run!("#{script}")
+    end
 
-          `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
+    ENV["DERAILED_STOP_VALID_COUNT"] ||= "10"
+    stop_valid_count = Integer(ENV["DERAILED_STOP_VALID_COUNT"])
 
-          exit(1)
-        end
+    times_significant = 0
+    DERAILED_SCRIPT_COUNT.times do |i|
+      puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+      branches_to_test.each do |branch, file|
+        Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
+        run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
       end
-    else
-      @app = Rack::MockRequest.new(DERAILED_APP)
-
-      def call_app
-        response = @app.get(PATH_TO_HIT, RACK_HTTP_HEADERS)
-        if response.status != 200
-          STDERR.puts "Couldn't call app. Bad request to #{PATH_TO_HIT}! Resulted in #{response.status} status."
-          STDERR.puts "\n\n***RESPONSE BODY***\n\n"
-          STDERR.puts response.body
-
-          FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
-
-          `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
+      times_significant += 1 if i >= 2 && stats.call.significant?
+      break if stop_valid_count != 0 && times_significant == stop_valid_count
+    end
 
-          exit(1)
-        end
-        response
+  ensure
+    if library_dir && current_library_branch
+      puts "Resetting git dir of #{library_dir.inspect} to #{current_library_branch.inspect}"
+      Dir.chdir(library_dir) do
+        run!("git checkout '#{current_library_branch}'")
       end
     end
-    if WARM_COUNT > 0
-      puts "Warming up app: #{WARM_COUNT} times"
-      WARM_COUNT.times { call_app }
-    end
+
+    stats.call.banner if stats
   end
 
   desc "hits the url TEST_COUNT times"
@@ -228,6 +172,11 @@ namespace :perf do
     require 'benchmark/ips'
 
     Benchmark.ips do |x|
+      x.warmup = Float(ENV["IPS_WARMUP"] || 2)
+      x.time = Float(ENV["IPS_TIME"] || 5)
+      x.suite = ENV["IPS_SUITE"] if ENV["IPS_SUITE"]
+      x.iterations = Integer(ENV["IPS_ITERATIONS"] || 1)
+
       x.report("ips") { call_app }
     end
   end
@@ -298,4 +247,10 @@ namespace :perf do
     puts ""
     puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
   end
+
+  def run!(cmd)
+    out = `#{cmd}`
+    raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+    out
+  end
 end

data/lib/derailed_benchmarks/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DerailedBenchmarks
-  VERSION = "1.3.6"
+  VERSION = "1.4.0"
 end

data/test/derailed_benchmarks/core_ext/kernel_require_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class KernelRequireTest < ActiveSupport::TestCase
@@ -25,6 +27,7 @@ class KernelRequireTest < ActiveSupport::TestCase
     assert_node_in_parent("child_one.rb", parent)
     child_two = assert_node_in_parent("child_two.rb", parent)
     assert_node_in_parent("relative_child", parent)
+    assert_node_in_parent("relative_child_two", parent)
     assert_node_in_parent("raise_child.rb", child_two)
   end
 end

data/test/derailed_benchmarks/require_tree_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class RequireTree < ActiveSupport::TestCase

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StatsFromDirTest < ActiveSupport::TestCase
+  test "that it works" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+
+    newest = stats.newest
+    oldest = stats.oldest
+
+    assert newest.average < oldest.average
+
+    assert_equal "winner", newest.name
+    assert_equal "loser", oldest.name
+
+    assert 3.6e-05 < stats.p_value
+    assert 3.8e-05 > stats.p_value
+    assert_equal true, stats.significant?
+
+    assert_equal "1.0062", stats.x_faster
+    assert_equal "0.6131", stats.percent_faster
+ end
+
+  test "banner faster" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    newest = stats.newest
+    oldest = stats.oldest
+
+    # Test fixture for banner
+    def stats.p_value
+      "0.000037"
+    end
+
+    def newest.average
+      10.5
+    end
+
+    def oldest.average
+      11.0
+    end
+
+    expected = <<-EOM
+[winner] "I am the new commit" - (10.5 seconds)
+  FASTER by:
+    1.0476x [older/newer]
+    4.5455% [(older - newer) / older * 100]
+[loser] "Old commit" - (11.0 seconds)
+EOM
+
+    actual = StringIO.new
+    stats.banner(actual)
+
+    assert_match expected, actual.string
+  end
+
+  test "banner slower" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "I am the new commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "Old commit", time: Time.now - 10, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    newest = stats.newest
+    oldest = stats.oldest
+
+    def oldest.average
+      10.5
+    end
+
+    def newest.average
+      11.0
+    end
+
+    expected = <<-EOM
+[loser] "I am the new commit" - (11.0 seconds)
+  SLOWER by:
+    0.9545x [older/newer]
+    -4.7619% [(older - newer) / older * 100]
+[winner] "Old commit" - (10.5 seconds)
+EOM
+
+    actual = StringIO.new
+    stats.banner(actual)
+
+    assert_match expected, actual.string
+  end
+
+  test "stats from samples with slightly different sizes" do
+    stats = DerailedBenchmarks::StatsFromDir.new({})
+    out = stats.students_t_test([100,101,102], [1,3])
+    assert out[:alternative]
+  end
+end

data/test/derailed_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class DerailedBenchmarksTest < ActiveSupport::TestCase

data/test/fixtures/require/child_one.rb

@@ -1,4 +1,4 @@
 class ChildOne
-  @retained = ""
+  @retained = +""
   50_000.times.map { @retained << "A" }
 end

data/test/fixtures/require/child_two.rb

@@ -1,5 +1,5 @@
 class ChildTwo
-  @retained = ""
+  @retained = +""
   200_000.times.map { @retained << "A" }
 end
 

data/test/fixtures/require/parent_one.rb

@@ -1,7 +1,8 @@
 class ParentOne
-  @retained = ""
+  @retained = +""
   1_000_000.times.map { @retained << "A" }
 end
 require File.expand_path('../child_one.rb', __FILE__)
 require File.expand_path('../child_two.rb', __FILE__)
 require_relative 'relative_child'
+require_relative File.expand_path('relative_child_two', __dir__)

data/test/fixtures/require/raise_child.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class RaiseChild
 end
 

data/test/fixtures/require/relative_child.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 class RelativeChild
 end