gitlab-derailed_benchmarks 1.6.1

data/lib/derailed_benchmarks/load_tasks.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+namespace :perf do
+  task :rails_load do
+    ENV["RAILS_ENV"] ||= "production"
+    ENV['RACK_ENV']  = ENV["RAILS_ENV"]
+    ENV["DISABLE_SPRING"] = "true"
+
+    ENV["SECRET_KEY_BASE"] ||= "foofoofoo"
+
+    ENV['LOG_LEVEL'] ||= "FATAL"
+
+    require 'rails'
+
+    puts "Booting: #{Rails.env}"
+
+    %W{ . lib test config }.each do |file|
+      $LOAD_PATH << File.expand_path(file)
+    end
+
+    require 'application'
+
+    Rails.env = ENV["RAILS_ENV"]
+
+    DERAILED_APP = Rails.application
+
+    if DERAILED_APP.respond_to?(:initialized?)
+      DERAILED_APP.initialize! unless DERAILED_APP.initialized?
+    else
+      DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
+    end
+
+    if !ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
+      if defined? ActiveRecord::Tasks::DatabaseTasks
+        ActiveRecord::Tasks::DatabaseTasks.create_current
+      else # Rails 3.2
+        raise "No valid database for #{ENV['RAILS_ENV']}, please create one" unless ActiveRecord::Base.connection.active?.inspect
+      end
+
+      ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
+      ActiveRecord::Migration.verbose         = true
+
+      if Rails.version.start_with? '6'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrate
+      elsif Rails.version.start_with? '5.2'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrate
+      else
+        ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+      end
+    end
+
+    DERAILED_APP.config.consider_all_requests_local = true
+  end
+
+  task :rack_load do
+    puts "You're not using Rails"
+    puts "You need to tell derailed how to boot your app"
+    puts "In your perf.rake add:"
+    puts
+    puts "namespace :perf do"
+    puts "  task :rack_load do"
+    puts "    # DERAILED_APP = your code here"
+    puts "  end"
+    puts "end"
+  end
+
+  task :setup do
+    if DerailedBenchmarks.gem_is_bundled?("railties")
+      Rake::Task["perf:rails_load"].invoke
+    else
+      Rake::Task["perf:rack_load"].invoke
+    end
+
+    WARM_COUNT  = (ENV['WARM_COUNT'] || 0).to_i
+    TEST_COUNT  = (ENV['TEST_COUNT'] || ENV['CNT'] || 1_000).to_i
+    PATH_TO_HIT = ENV["PATH_TO_HIT"] || ENV['ENDPOINT'] || "/"
+    puts "Endpoint: #{ PATH_TO_HIT.inspect }"
+
+    HTTP_HEADER_PREFIX = "HTTP_".freeze
+    RACK_HTTP_HEADERS = ENV.select { |key| key.start_with?(HTTP_HEADER_PREFIX) }
+
+    HTTP_HEADERS = RACK_HTTP_HEADERS.keys.inject({}) do |hash, rack_header_name|
+      # e.g. "HTTP_ACCEPT_CHARSET" -> "Accept-Charset"
+      header_name = rack_header_name[HTTP_HEADER_PREFIX.size..-1].split("_").map(&:downcase).map(&:capitalize).join("-")
+      hash[header_name] = RACK_HTTP_HEADERS[rack_header_name]
+      hash
+    end
+    puts "HTTP headers: #{HTTP_HEADERS}" unless HTTP_HEADERS.empty?
+
+    CURL_HTTP_HEADER_ARGS = HTTP_HEADERS.map { |http_header_name, value| "-H \"#{http_header_name}: #{value}\"" }.join(" ")
+
+    require 'rack/test'
+    require 'rack/file'
+
+    DERAILED_APP = DerailedBenchmarks.add_auth(Object.class_eval { remove_const(:DERAILED_APP) })
+    if server = ENV["USE_SERVER"]
+      @port = (3000..3900).to_a.sample
+      puts "Port: #{ @port.inspect }"
+      puts "Server: #{ server.inspect }"
+      thread = Thread.new do
+        Rack::Server.start(app: DERAILED_APP, :Port => @port, environment: "none", server: server)
+      end
+      sleep 1
+
+      def call_app(path = File.join("/", PATH_TO_HIT))
+        cmd = "curl #{CURL_HTTP_HEADER_ARGS} 'http://localhost:#{@port}#{path}' -s --fail 2>&1"
+        response = `#{cmd}`
+        unless $?.success?
+          STDERR.puts "Couldn't call app."
+          STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
+
+          FileUtils.mkdir_p("tmp")
+          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+
+          `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
+
+          exit(1)
+        end
+      end
+    else
+      @app = Rack::MockRequest.new(DERAILED_APP)
+
+      def call_app
+        response = @app.get(PATH_TO_HIT, RACK_HTTP_HEADERS)
+        if response.status != 200
+          STDERR.puts "Couldn't call app. Bad request to #{PATH_TO_HIT}! Resulted in #{response.status} status."
+          STDERR.puts "\n\n***RESPONSE BODY***\n\n"
+          STDERR.puts response.body
+
+          FileUtils.mkdir_p("tmp")
+          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+
+          `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
+
+          exit(1)
+        end
+        response
+      end
+    end
+    if WARM_COUNT > 0
+      puts "Warming up app: #{WARM_COUNT} times"
+      WARM_COUNT.times { call_app }
+    end
+  end
+end

data/lib/derailed_benchmarks/require_tree.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Tree structure used to store and sort require memory costs
+# RequireTree.new('get_process_mem')
+module DerailedBenchmarks
+  class RequireTree
+    REQUIRED_BY = {}
+
+    attr_reader   :name
+    attr_writer   :cost
+    attr_accessor :parent
+
+    def initialize(name)
+      @name     = name
+      @children = {}
+    end
+
+    def <<(tree)
+      @children[tree.name.to_s] = tree
+      tree.parent = self
+      (REQUIRED_BY[tree.name.to_s] ||= []) << self.name
+    end
+
+    def [](name)
+      @children[name.to_s]
+    end
+
+    # Returns array of child nodes
+    def children
+      @children.values
+    end
+
+    def cost
+      @cost || 0
+    end
+
+    # Returns sorted array of child nodes from Largest to Smallest
+    def sorted_children
+      children.sort { |c1, c2| c2.cost <=> c1.cost }
+    end
+
+    def to_string
+      str = +"#{name}: #{cost.round(4)} MiB"
+      if parent && REQUIRED_BY[self.name.to_s]
+        names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
+        if names.any?
+          str << " (Also required by: #{ names.first(2).join(", ") }"
+          str << ", and #{names.count - 2} others" if names.count > 3
+          str << ")"
+        end
+      end
+      str
+    end
+
+    # Recursively prints all child nodes
+    def print_sorted_children(level = 0, out = STDOUT)
+      return if cost < ENV['CUT_OFF'].to_f
+      out.puts "  " * level + self.to_string
+      level += 1
+      sorted_children.each do |child|
+        child.print_sorted_children(level, out)
+      end
+    end
+  end
+end

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'bigdecimal'
+require 'statistics'
+
+module DerailedBenchmarks
+  # A class used to read several benchmark files
+  # it will parse each file, then sort by average
+  # time of benchmarks. It can be used to find
+  # the fastest and slowest examples and give information
+  # about them such as what the percent difference is
+  # and if the results are statistically significant
+  #
+  # Example:
+  #
+  #   branch_info = {}
+  #   branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+  #   branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+  #   stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
+  #
+  #   stats.newest.average  # => 10.5
+  #   stats.oldest.average  # => 11.0
+  #   stats.significant?    # => true
+  #   stats.x_faster        # => "1.0476"
+  class StatsFromDir
+    FORMAT = "%0.4f"
+    attr_reader :stats, :oldest, :newest
+
+    def initialize(hash)
+      @files = []
+
+      hash.each do |branch, info_hash|
+        file = info_hash.fetch(:file)
+        desc = info_hash.fetch(:desc)
+        time = info_hash.fetch(:time)
+        @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
+      end
+      @files.sort_by! { |f| f.time }
+      @oldest = @files.first
+      @newest = @files.last
+    end
+
+    def call
+      @files.each(&:call)
+
+      stats_95 = statistical_test(confidence: 95)
+
+      # If default check is good, see if we also pass a more rigorous test
+      # if so, then use the more rigourous test
+      if stats_95[:alternative]
+        stats_99 = statistical_test(confidence: 99)
+        @stats = stats_99 if stats_99[:alternative]
+      end
+      @stats ||= stats_95
+
+      self
+    end
+
+    def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
+      StatisticalTest::KSTest.two_samples(
+        group_one: series_1,
+        group_two: series_2,
+        alpha: (100 - confidence) / 100.0
+      )
+    end
+
+    def significant?
+      @stats[:alternative]
+    end
+
+    def d_max
+      @stats[:d_max].to_f
+    end
+
+    def d_critical
+      @stats[:d_critical].to_f
+    end
+
+    def x_faster
+      (oldest.median/newest.median).to_f
+    end
+
+    def faster?
+      newest.median < oldest.median
+    end
+
+    def percent_faster
+      (((oldest.median - newest.median) / oldest.median).to_f  * 100)
+    end
+
+    def change_direction
+      if faster?
+        "FASTER 🚀🚀🚀"
+      else
+        "SLOWER 🐢🐢🐢"
+      end
+    end
+
+    def align
+      " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
+    end
+
+    def banner(io = Kernel)
+      io.puts
+      if significant?
+        io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
+      else
+        io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
+      end
+      io.puts
+      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
+      io.puts "  #{change_direction} by:"
+      io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
+      io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
+      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
+      io.puts
+      io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
+      io.puts "Samples: #{newest.values.length}"
+      io.puts
+      io.puts "Test type: Kolmogorov Smirnov"
+      io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
+      io.puts "Is significant? (max > critical): #{significant?}"
+      io.puts "D critical: #{d_critical}"
+      io.puts "D max: #{d_max}"
+      io.puts
+    end
+  end
+end

data/lib/derailed_benchmarks/stats_in_file.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module DerailedBenchmarks
+  # A class for reading in benchmark results
+  # and converting them to numbers for comparison
+  #
+  # Example:
+  #
+  #  puts `cat muhfile.bench.txt`
+  #
+  #    9.590142   0.831269  10.457801 ( 10.0)
+  #    9.836019   0.837319  10.728024 ( 11.0)
+  #
+  #  x = StatsForFile.new(name: "muhcommit", file: "muhfile.bench.txt", desc: "I made it faster", time: Time.now)
+  #  x.values  #=> [11.437769, 11.792425]
+  #  x.average # => 10.5
+  #  x.name    # => "muhfile"
+  class StatsForFile
+    attr_reader :name, :values, :desc, :time
+
+    def initialize(file:, name:, desc: "", time: )
+      @file = Pathname.new(file)
+      FileUtils.touch(@file)
+
+      @name = name
+      @desc = desc
+      @time = time
+    end
+
+    def call
+      load_file!
+
+      @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
+      @average = values.inject(:+) / values.length
+    end
+
+    def median
+      @median.to_f
+    end
+
+    def average
+      @average.to_f
+    end
+
+    private def load_file!
+      @values = []
+      @file.each_line do |line|
+        line.match(/\( +(\d+\.\d+)\)/)
+        begin
+          values << BigDecimal($1)
+        rescue => e
+          raise e, "Problem with file #{@file.inspect}:\n#{@file.read}\n#{e.message}"
+        end
+      end
+
+      values.sort!
+      values.freeze
+    end
+  end
+end

data/lib/derailed_benchmarks/tasks.rb

@@ -0,0 +1,292 @@
+require_relative 'load_tasks'
+
+namespace :perf do
+  desc "runs the performance test against two most recent commits of the current app"
+  task :app do
+    ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
+    Rake::Task["perf:library"].invoke
+  end
+
+  desc "runs the same test against two different branches for statistical comparison"
+  task :library do
+    begin
+      DERAILED_SCRIPT_COUNT = (ENV["DERAILED_SCRIPT_COUNT"] ||= "200").to_i
+      ENV["TEST_COUNT"] ||= "200"
+
+      raise "test count must be at least 2, is set to #{DERAILED_SCRIPT_COUNT}" if DERAILED_SCRIPT_COUNT < 2
+      script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
+
+      if ENV["DERAILED_PATH_TO_LIBRARY"]
+        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
+      else
+        library_dir = DerailedBenchmarks.rails_path_on_disk
+      end
+
+      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
+
+      # Use either the explicit SHAs when present or grab last two SHAs from commit history
+      # if only one SHA is given, then use it and the last SHA from commit history
+      branch_names = []
+      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
+      if branch_names.length < 2
+        Dir.chdir(library_dir) do
+          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
+
+          branches = run!('git log --format="%H" -n 2').chomp.split($/)
+          if branch_names.empty?
+            branch_names = branches
+          else
+            branches.shift
+            branch_names << branches.shift
+          end
+        end
+      end
+
+      current_library_branch = ""
+      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
+
+      out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
+      out_dir.mkpath
+
+      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
+      branch_info = {}
+      branch_to_sha = {}
+
+      branches_to_test.each do |branch, file|
+        Dir.chdir(library_dir) do
+          run!("git checkout '#{branch}'")
+          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
+          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
+          short_sha   = run!("git rev-parse --short HEAD").strip
+          branch_to_sha[branch] = short_sha
+
+          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
+        end
+        run!("#{script}")
+      end
+
+      puts
+      puts
+      branches_to_test.each.with_index do |(branch, _), i|
+        short_sha = branch_to_sha[branch]
+        desc      = branch_info[short_sha][:desc]
+        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
+      end
+      puts
+      puts
+
+      raise "SHAs to test must be different" if branch_info.length == 1
+      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
+      puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]
+
+      DERAILED_SCRIPT_COUNT.times do |i|
+        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+        branches_to_test.each do |branch, file|
+          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
+          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
+        end
+
+        if (i % 50).zero?
+          puts "Intermediate result"
+          stats.call.banner
+          puts "Continuing execution"
+        end
+      end
+
+    ensure
+      if library_dir && current_library_branch
+        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
+        Dir.chdir(library_dir) do
+          run!("git checkout '#{current_library_branch}'")
+        end
+      end
+
+      if stats
+        stats.call.banner
+
+        result_file = out_dir + "results.txt"
+        File.open(result_file, "w") do |f|
+          stats.banner(f)
+        end
+
+        puts "Output: #{result_file.to_s}"
+      end
+    end
+  end
+
+  desc "hits the url TEST_COUNT times"
+  task :test => [:setup] do
+    require 'benchmark'
+
+    Benchmark.bm { |x|
+      x.report("#{TEST_COUNT} derailed requests") {
+        TEST_COUNT.times {
+          call_app
+        }
+      }
+    }
+  end
+
+  desc "stackprof"
+  task :stackprof => [:setup] do
+    # [:wall, :cpu, :object]
+    begin
+      require 'stackprof'
+    rescue LoadError
+      raise "Add stackprof to your gemfile to continue `gem 'stackprof', group: :development`"
+    end
+    TEST_COUNT = (ENV["TEST_COUNT"] ||= "100").to_i
+    file = "tmp/#{Time.now.iso8601}-stackprof-cpu-myapp.dump"
+    StackProf.run(mode: :cpu, out: file) do
+      Rake::Task["perf:test"].invoke
+    end
+    cmd = "stackprof #{file}"
+    puts "Running `#{cmd}`. Execute `stackprof --help` for more info"
+    puts `#{cmd}`
+  end
+
+  task :kernel_require_patch do
+    require 'derailed_benchmarks/core_ext/kernel_require.rb'
+  end
+
+  desc "show memory usage caused by invoking require per gem"
+  task :mem => [:kernel_require_patch, :setup] do
+    puts "## Impact of `require <file>` on RAM"
+    puts
+    puts "Showing all `require <file>` calls that consume #{ENV['CUT_OFF']} MiB or more of RSS"
+    puts "Configure with `CUT_OFF=0` for all entries or `CUT_OFF=5` for few entries"
+
+    puts "Note: Files only count against RAM on their first load."
+    puts "      If multiple libraries require the same file, then"
+    puts "       the 'cost' only shows up under the first library"
+    puts
+
+    call_app
+
+    TOP_REQUIRE.print_sorted_children
+  end
+
+  desc "outputs memory usage over time"
+  task :mem_over_time => [:setup] do
+    require 'get_process_mem'
+    puts "PID: #{Process.pid}"
+    ram = GetProcessMem.new
+    @keep_going = true
+    begin
+      unless ENV["SKIP_FILE_WRITE"]
+        ruby = `ruby -v`.chomp
+        FileUtils.mkdir_p("tmp")
+        file = File.open("tmp/#{Time.now.iso8601}-#{ruby}-memory-#{TEST_COUNT}-times.txt", 'w')
+        file.sync = true
+      end
+
+      ram_thread = Thread.new do
+        while @keep_going
+          mb = ram.mb
+          STDOUT.puts mb
+          file.puts mb unless ENV["SKIP_FILE_WRITE"]
+          sleep 5
+        end
+      end
+
+      TEST_COUNT.times {
+        call_app
+      }
+    ensure
+      @keep_going = false
+      ram_thread.join
+      file.close unless ENV["SKIP_FILE_WRITE"]
+    end
+  end
+
+  task :ram_over_time do
+    raise "Use mem_over_time"
+  end
+
+  desc "iterations per second"
+  task :ips => [:setup] do
+    require 'benchmark/ips'
+
+    Benchmark.ips do |x|
+      x.warmup = Float(ENV["IPS_WARMUP"] || 2)
+      x.time = Float(ENV["IPS_TIME"] || 5)
+      x.suite = ENV["IPS_SUITE"] if ENV["IPS_SUITE"]
+      x.iterations = Integer(ENV["IPS_ITERATIONS"] || 1)
+
+      x.report("ips") { call_app }
+    end
+  end
+
+  desc "outputs GC::Profiler.report data while app is called TEST_COUNT times"
+  task :gc => [:setup] do
+    GC::Profiler.enable
+    TEST_COUNT.times { call_app }
+    GC::Profiler.report
+    GC::Profiler.disable
+  end
+
+  desc "outputs allocated object diff after app is called TEST_COUNT times"
+  task :allocated_objects => [:setup] do
+    call_app
+    GC.start
+    GC.disable
+    start = ObjectSpace.count_objects
+    TEST_COUNT.times { call_app }
+    finish = ObjectSpace.count_objects
+    GC.enable
+    finish.each do |k,v|
+      puts k => (v - start[k]) / TEST_COUNT.to_f
+    end
+  end
+
+
+  desc "profiles ruby allocation"
+  task :objects => [:setup] do
+    require 'memory_profiler'
+    call_app
+    GC.start
+
+    num = Integer(ENV["TEST_COUNT"] || 1)
+    opts = {}
+    opts[:ignore_files] = /#{ENV['IGNORE_FILES_REGEXP']}/ if ENV['IGNORE_FILES_REGEXP']
+    opts[:allow_files]  = "#{ENV['ALLOW_FILES']}"         if ENV['ALLOW_FILES']
+
+    puts "Running #{num} times"
+    report = MemoryProfiler.report(opts) do
+      num.times { call_app }
+    end
+    report.pretty_print
+  end
+
+  desc "heap analyzer"
+  task :heap => [:setup] do
+    require 'objspace'
+
+    file_name = "tmp/#{Time.now.iso8601}-heap.dump"
+    FileUtils.mkdir_p("tmp")
+    ObjectSpace.trace_object_allocations_start
+    puts "Running #{ TEST_COUNT } times"
+    TEST_COUNT.times {
+      call_app
+    }
+    GC.start
+
+    puts "Heap file generated: #{ file_name.inspect }"
+    ObjectSpace.dump_all(output: File.open(file_name, 'w'))
+
+    require 'heapy'
+
+    Heapy::Analyzer.new(file_name).analyze
+
+    puts ""
+    puts "Run `$ heapy --help` for more options"
+    puts ""
+    puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
+  end
+
+  def run!(cmd)
+    out = `#{cmd}`
+    raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+    out
+  end
+end