perf_check 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e429268ea248d238130f873494e04bdf8b74a3fd
-  data.tar.gz: 49cd55f1113f6f31492a84c273d58761d1ce824f
+  metadata.gz: 55dd84ea36a6406e768ed5581acf65b9f4b6aa50
+  data.tar.gz: fb5b16a1aea5f51c75af09c9843d4bd3a063ff30
 SHA512:
-  metadata.gz: 263a12324d08e02724e6ef92703a02a0ea13e5cf07c846550189c628aaba177c5662492089745d6f172e5cab2ef5be7e46039ee6ed8100f6ebc8cf1f6bcd9872
-  data.tar.gz: 56b851ddfd60cc6192c04cedafc2c715983cf01c930135da2881d2bacc5b62edc2eaac70092c68e2f2b14c404f9003f4596c8ba1ce893ba62a771669225138f6
+  metadata.gz: 1377387b9ef80a3bbbc9128b4461bf89ffb189fcbee5f2f685becee729681affb51a193725a5bcc414f0ac9eb57176c010fb89bd4a1a403ee8194f33aadbabc0
+  data.tar.gz: 64d8c600ad45a422698f31717e0612a150f7899d7252cf60c4c9c9fb70f531e13445605a9783c126dcaa1c30b06d9c5706ccf19bab9651c1e612aa24b7f78ba5

data/bin/perf_check

@@ -8,7 +8,8 @@ options.login = nil
 options.number_of_requests = 10
 options.reference = 'master'
 options.http_statuses = [200]
-
+options.verify_responses = false
+options.caching = true
 
 opts = OptionParser.new do |opts|
   opts.banner = "Usage: perf_check [options] [route ...]"
@@ -52,7 +53,11 @@ opts = OptionParser.new do |opts|
   end
 
   opts.on('--no-caching', 'Do not enable fragment caching') do
-    ENV['PERF_CHECK_NOCACHING'] = 'true'
+    options.caching = false
+  end
+
+  opts.on('--fail-fast', '-f', 'Bail immediately on non-200 HTTP response') do
+    options[:fail_fast?] = true
   end
 
   opts.on('--302-success', 'Consider HTTP 302 code a successful request') do
@@ -63,6 +68,29 @@ opts = OptionParser.new do |opts|
     options.http_statuses.delete(302)
   end
 
+  opts.separator "\nMisc"
+  opts.on('--input FILE', '-i') do |input|
+    File.readlines(input).each do |resource|
+      ARGV << resource.strip
+    end
+  end
+
+  opts.on('--verify-responses',
+          'Check whether there is a diff between the responses of this and the reference branch') do
+    options.verify_responses = true
+  end
+
+  opts.on('--brief', '-b') do
+    options.brief = true
+  end
+
+  opts.on('--diff') do
+    options.diff = true
+    options.brief = true
+    options.verify_responses = true
+    options.number_of_requests = 1
+  end
+
   opts.separator ''
   opts.separator <<EOF
 Usage examples:
@@ -70,6 +98,7 @@ Usage examples:
      perf_check /user/45/posts
      perf_check /user/45/posts -n5
      perf_check /user/45/posts --standard
+     perf_check /user/45/posts --admin
 
   Benchmark against a specific commit
      perf_check /user/45/posts -r 0123abcdefg
@@ -77,11 +106,20 @@ Usage examples:
 
   Benchmark the changes in the working tree
      perf_check /user/45/posts -r HEAD
+
+  Benchmark and diff the output against master
+     perf_check /user/45/posts --verify-responses
+
+  Just diff the output on your branch with master
+     perf_check /user/45/posts --diff
+
+  Diff a bunch of urls listed in a file (newline seperated)
+    perf_check --diff --input FILE
 EOF
 end
 opts.parse!
 
-PerfCheck.require_rails
+PerfCheck.require_rails(options)
 
 ARGV.each do |route|
   perf_check.add_test_case(route)
@@ -94,7 +132,11 @@ end
 perf_check.sanity_check
 perf_check.run
 
-perf_check.print_results
+if options.brief
+  perf_check.print_brief_results
+else
+  perf_check.print_results
+end
 
 #            _______________________
 #           < You made it faster!!! >

data/lib/perf_check/git.rb

@@ -32,7 +32,6 @@ class PerfCheck
         system('git stash -q >/dev/null')
         abort("Problem with git stash! Bailing...") unless $?.success?
         at_exit do
-          system('git checkout .')
           Git.pop
         end
       end

data/lib/perf_check/railtie.rb

@@ -21,6 +21,10 @@ class PerfCheck
         config = ActiveSupport::Configurable::Configuration
         config.send(:define_method, :perform_caching){ fragment_caching }
       end
+
+      if ENV['PERF_CHECK'] && ENV['PERF_CHECK_VERIFICATION']
+        PerfCheck::Server.seed_random!
+      end
     end
   end
 end

data/lib/perf_check/server.rb

@@ -12,7 +12,7 @@ class PerfCheck
 
     def self.authorization_action(method, login_route, &block)
       Rails.application.reload_routes!
-      p = Rails.application.routes.recognize_path(login_route, :method => method)
+      p = PerfCheck::Server.recognize_path(login_route, :method => method)
       controller = "#{p[:controller]}_controller".classify.constantize
       action = p[:action]
 
@@ -35,6 +35,49 @@ class PerfCheck
       end
     end
 
+    def self.mounted_path(engine)
+      route = Rails.application.routes.routes.detect do |route|
+        route.app == engine
+      end
+      route && route.path
+    end
+
+    def self.recognize_path(path, opts={})
+      Rails::Engine.subclasses.each do |engine|
+        if match = mounted_path(engine) =~ path
+          path = path.sub(match.to_s, '')
+          return engine.routes.recognize_path(path, opts)
+        end
+      end
+      Rails.application.routes.recognize_path(path, opts)
+    end
+
+    def self.seed_random!
+      # Seed random
+      srand(1)
+
+      # SecureRandom cannot be seeded, so we have to monkey patch it instead :\
+      def SecureRandom.hex(n=16)
+        '4' * n
+      end
+
+      def SecureRandom.random_bytes(n=16)
+        '4' * n
+      end
+
+      def SecureRandom.random_number(n=0)
+        n > 4 ? 4 : n
+      end
+
+      def SecureRandom.urlsafe_base64(n=16, padding=false)
+        '4' * (4*n / 3)
+      end
+
+      def SecureRandom.uuid
+        "00000000-0000-0000-0000-000000000004"
+      end
+    end
+
     def self.sign_cookie_data(key, data, opts={})
       opts[:serializer] ||= Marshal
       secret = Rails.application.config.secret_token
@@ -116,10 +159,10 @@ class PerfCheck
 
     def restart
       if !@running
-        $stderr.print "starting rails..."
+        $stderr.print "starting rails...\n"
         start
       else
-        $stderr.print "re-starting rails..."
+        $stderr.print "re-starting rails...\n"
         exit
         start
       end

data/lib/perf_check/test_case.rb

@@ -1,17 +1,18 @@
 # coding: utf-8
 
+require 'diffy'
+
 class PerfCheck
   class TestCase
     attr_accessor :resource, :controller, :action, :format
-    attr_accessor :latencies, :cookie
-    attr_accessor :this_latencies, :reference_latencies
+    attr_accessor :cookie, :this_response, :reference_response
+    attr_accessor :this_profiles, :reference_profiles
 
     def initialize(route)
-      params = Rails.application.routes.recognize_path(route)
+      params = PerfCheck::Server.recognize_path(route)
 
-      self.this_latencies = []
-      self.reference_latencies = []
-      self.latencies = this_latencies
+      self.this_profiles = []
+      self.reference_profiles = []
 
       self.controller = params[:controller].split('/')[-1]
       self.action = params[:action]
@@ -19,11 +20,20 @@ class PerfCheck
       self.resource = route
     end
 
+    def switch_to_reference_context
+      @context = :reference
+    end
+
     def run(server, options)
-      print("\t"+'request #'.underline)
-      print("  "+'latency'.underline)
-      print("   "+'server rss'.underline)
-      puts("   "+'profiler data'.underline)
+      unless options.diff
+        print("\t"+'request #'.underline)
+        print("  "+'latency'.underline)
+        print("   "+'server rss'.underline)
+        print("   "+'status'.underline)
+        puts("   "+'profiler data'.underline)
+      end
+
+      profiles = (@context == :reference) ? reference_profiles : this_profiles
 
       headers = {'Cookie' => "#{cookie}"}
       unless self.format
@@ -35,31 +45,47 @@ class PerfCheck
         end
 
         unless options.http_statuses.include? profile.response_code
-          File.open("tmp/perf_check/failed_request.html", 'w') do |error_dump|
-            error_dump.write(profile.response_body)
+          if options.fail_fast?
+            File.open("tmp/perf_check/failed_request.html", 'w') do |error_dump|
+              error_dump.write(profile.response_body)
+            end
+            error = sprintf("\t%2i:\tFAILED! (HTTP %d)", i, profile.response_code)
+            puts(error.red.bold)
+            puts("\t   The server responded with a non-2xx status for this request.")
+            puts("\t   The response has been written to tmp/perf_check/failed_request.html")
+            exit(1)
           end
-          error = sprintf("\t%2i:\tFAILED! (HTTP %d)", i, profile.response_code)
-          puts(error.red.bold)
-          puts("\t   The server responded with a non-2xx status for this request.")
-          puts("\t   The response has been written to tmp/perf_check/failed_request.html")
-          exit(1)
         end
 
         next if i.zero?
-        printf("\t%2i:\t   %.1fms   %4dMB\t  %s\n",
-               i, profile.latency, server.mem, profile.profile_url)
 
-        self.latencies << profile.latency
+        if options.verify_responses
+          if i == 1
+            if @context == :reference
+              self.reference_response = profile.response_body
+            else
+              self.this_response = profile.response_body
+            end
+          end
+        end
+
+        unless options.diff
+          printf("\t%2i:\t   %.1fms   %4dMB\t  %s\t   %s\n",
+                 i, profile.latency, server.mem,
+                 profile.response_code, profile.profile_url)
+        end
+
+        profiles << profile
       end
-      puts
+      puts unless options.diff # pretty!
     end
 
     def this_latency
-      this_latencies.inject(0.0, :+) / this_latencies.size
+      this_profiles.map(&:latency).inject(0.0, :+) / this_profiles.size
     end
 
     def reference_latency
-      reference_latencies.inject(0.0, :+) / reference_latencies.size
+      reference_profiles.map(&:latency).inject(0.0, :+) / reference_profiles.size
     end
 
     def latency_difference
@@ -70,6 +96,22 @@ class PerfCheck
       reference_latency / this_latency
     end
 
+    def response_diff
+      diff = Diffy::Diff.new(this_response, reference_response,
+                             :diff => PerfCheck.diff_options)
+      if diff.to_s.empty?
+        OpenStruct.new(:changed? => false)
+      else
+        FileUtils.mkdir_p("#{Rails.root}/tmp/perf_check/diffs")
+        file = `mktemp -u "#{Rails.root}/tmp/perf_check/diffs/XXXXXXXXXX"`.strip
+        [:text, :html].each do |format|
+          ext = {:text => 'diff', :html => 'html'}[format]
+          File.open("#{file}.#{ext}", 'w'){ |f| f.write(diff.to_s(format)) }
+        end
+        OpenStruct.new(:changed? => true, :file => "#{file}.diff")
+      end
+    end
+
     def eql?(test)
       resource == test.resource
     end

data/lib/perf_check.rb

@@ -11,8 +11,19 @@ require 'colorize'
 class PerfCheck
   attr_accessor :options, :server, :test_cases
 
-  def self.require_rails
+  def self.diff_options
+    @@diff_options ||=
+      ['-U3', '--ignore-matching-lines=/mini-profiler-resources/includes.js']
+  end
+
+  def self.require_rails(options)
     ENV['PERF_CHECK'] = '1'
+    if options.verify_responses
+      ENV['PERF_CHECK_VERIFICATION'] = '1'
+    end
+    if !options.caching
+      ENV['PERF_CHECK_NOCACHING'] = '1'
+    end
 
     app_root = Dir.pwd
     until app_root == '/' || File.exist?("#{app_root}/config/application.rb")
@@ -57,28 +68,59 @@ class PerfCheck
       if i == 1
         Git.stash_if_needed
         Git.checkout_reference(options.reference)
-        test_cases.each{ |x| x.latencies = x.reference_latencies }
+        test_cases.each{ |x| x.switch_to_reference_context }
       end
 
-      test_cases.each do |test|
-        server.restart
+      server.restart
+      test_cases.each_with_index do |test, i|
+        server.restart unless i.zero? || options.diff
 
         if options.login
           test.cookie = server.login(options.login, test)
         end
 
-        puts("\n\nBenchmarking #{test.resource}:")
+        if options.diff
+          puts "Issuing #{test.resource}"
+        else
+          puts("\nBenchmarking #{test.resource}:")
+        end
         test.run(server, options)
       end
     end
   end
 
+
+  def print_diff_results(diff)
+    if diff.changed?
+      print(" Diff: #{diff.file}".bold.light_red)
+    else
+      print(" Diff: Output is identical!".bold.light_green)
+    end
+  end
+
+  def print_brief_results
+    test_cases.each do |test|
+      print(test.resource.ljust(40) + ': ')
+
+      codes = (test.this_profiles+test.reference_profiles).map(&:response_code).uniq
+      print("(HTTP "+codes.join(',')+") ")
+
+      printf('%.1fms', test.this_latency)
+
+      puts && next if test.reference_profiles.empty?
+
+      print(sprintf(' (%+5.1fms)', test.latency_difference).bold)
+      print_diff_results(test.response_diff) if options.verify_responses
+      puts
+    end
+  end
+
   def print_results
     puts("==== Results ====")
     test_cases.each do |test|
-      puts(test.resource)
+      puts(test.resource.bold)
 
-      if test.reference_latencies.empty?
+      if test.reference_profiles.empty?
         printf("your branch: ".rjust(15)+"%.1fms\n", test.this_latency)
         next
       end
@@ -110,6 +152,8 @@ class PerfCheck
       puts("master: ".rjust(15)     + "#{master_latency}")
       puts("your branch: ".rjust(15)+ "#{this_latency}")
       puts(("change: ".rjust(15)     + "#{formatted_change}").bold.send(color))
+
+      print_diff_results(test.response_diff) if options.verify_responses
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perf_check
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - rubytune
@@ -24,6 +24,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.7.3
+- !ruby/object:Gem::Dependency
+  name: diffy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.5
 description: 
 email: 
 executables: