log_replayer 0.0.4

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem         # Don't check in built gems.
+.yardoc       # Yard metadata.
+Gemfile.lock  # Gems shouldn't lock gem versions. Apps that use them should.
+_yardoc       # More yard metadata.
+doc/          # Don't check in yard-generated docs. See yard-doc.ooyala.com.
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in log_replayer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1 @@
+Copyright (c) 2012, Ooyala Inc

data/README.md

@@ -0,0 +1,3 @@
+# LogReplayer
+
+Replays log files, which can be used as load testing.

data/lib/log_replayer.rb

@@ -0,0 +1,169 @@
+#!/usr/bin/env ruby
+#
+# Load test SAS by replaying requests in a multiple of real-time.
+#
+# TODO:
+#   - Add configurability.  Lots of it.
+#   - Instead of spewing lots of data, set thresholds and log requests that go over them.
+#     Or log lots of data in a gnuplot/csv friendly format for comparing graphs
+#     Or give back useful average/percentile metrics
+#   - Find the "maximum stable multiplier"; replay requests at increasing speed until we start seeing delays
+#   - Replay at maximum speed (effectively an infinite multiplier)
+#   - Automatically pull down useful graphs from Ganglia or machine metrics from Hastur
+#   - Figure out why we don't need to sign requests, fix that, and then resign them
+
+
+require "em-synchrony"
+require "em-synchrony/em-http"
+require "em-synchrony/fiber_iterator"
+require "time"
+require "trollop"
+
+module LogReplayer
+  class Replayer
+    IP_REGEX = "[^-]*" # Can include multiple comma (with optional spaces) ips or "unknown"
+    DATE_REGEX = "[0-9]{1,2}/[A-Z][a-z]{2}/[0-9]{4} *[0-9]{1,2}:[0-9]{2}:[0-9]{2}"
+    LOG_LINE_REGEX = /^#{IP_REGEX} - - \[(#{DATE_REGEX})\] "GET ([^"]*) HTTP\/1.[01]"/
+
+    EXCLUDE_ROUTES = %w{/health_check}.map { |regex| /^#{regex}/ }
+
+    def initialize(options)
+      @host = options[:host]
+      @concurrency = options[:concurrency]
+      @speed = options[:speed]
+      @csv_output = options[:csv_output]
+      @summary_output = options[:summary_output]
+      @profiler = options[:profiler]
+    end
+
+    def load_request_routes(file_name)
+      @requests = []
+      File.open(file_name) do |log_file|
+        log_file.each_line do |log_line|
+          @requests << parse_requests(log_line)
+        end
+      end
+      @requests.compact!
+      puts @requests.length
+    end
+
+    # returns hash of { :time, :route, :action, :request_options }
+    # override this in subclass to work support different log formats
+    def parse_requests(log_line)
+      match = log_line.match(LOG_LINE_REGEX)
+      return nil unless match
+      date, route = match[1..2]
+      return nil if EXCLUDE_ROUTES.any? { |exclude| route.match(exclude) }
+      { :time => Time.parse(date), :route => route, :action => :get, :request_options => {} }
+    end
+
+    # returns string with additional string for the summary, override for different statusz implementations
+    def additional_summary
+      statusz = EM::HttpRequest.new("http://#{@host}/statusz").get
+      [/Branch.*/, /Ruby Version.*/].map {|x| x.match(statusz.response)}.compact.join(" -- ")
+    end
+
+    # Replay logs in (a multiple of) real time
+    def replay_requests
+      log_start_time = @requests[0][:time]
+      replay_start_time = Time.now
+      count = 0
+      csv_file = File.open(@csv_output, "w") if @csv_output
+      csv_file.puts "id,status,processing time,start delay" if @csv_output
+
+      error_count = 0
+      status_counts = Hash.new(0)
+      total_time = 0
+      max_request_processing_time = 0
+      min_request_processing_time = nil
+      app_build = nil
+      ruby_version = nil
+      summary_header = ''
+      EM.synchrony do
+        # __start__ and __stop__ are for rack-perftools_profiler.
+        EventMachine::HttpRequest.new("http://#{@host}/__start__").get if @profiler
+        EM::Synchrony::FiberIterator.new(@requests, @concurrency).each do |request|
+          next unless [:get, :post, :patch, :put, :delete].include? request[:action]
+          # Track which request this is.  Note that count is shared between fibers, so will likely be changed by
+          # another fiber before it's used.  So store it in the block-local mycount.
+          count += 1
+          mycount = count
+
+          # Figure out when the request should start, and sleep until then
+          request_offset = request[:time] - log_start_time
+          replay_time = replay_start_time + request_offset / @speed
+          sleep_time = replay_time - Time.now
+          EM::Synchrony.sleep(sleep_time) if sleep_time > 0
+
+          # Send the request, and track how long it takes to complete
+          url = "http://#{@host}#{request[:route]}"
+          request_start_time = Time.now
+          client = EventMachine::HttpRequest.new(url, :connect_timeout => 15, :inactivity_timeout => 60)
+          response = client.public_send(request[:action], request[:request_options])
+
+          request_processing_time = Time.now - request_start_time
+          if (request_processing_time > max_request_processing_time)
+            max_request_processing_time = request_processing_time
+          end
+          if min_request_processing_time.nil? || min_request_processing_time > request_processing_time
+            min_request_processing_time = request_processing_time
+          end
+          status = response.response_header.http_status
+
+          if csv_file
+            csv_file.puts "#{mycount},#{status},#{request_processing_time},#{request_start_time - replay_time}"
+          end
+          total_time += request_processing_time
+          status_counts[status] += 1
+          error_count += 1 if response.error
+          STDERR.puts "#{mycount}: Error: #{response.error}" if response.error
+          unless status == 200
+            STDERR.puts "#{mycount}: received status code: #{status}, body: #{response.response}"
+          end
+        end
+        EventMachine::HttpRequest.new("http://#{@host}/__stop__").get if @profiler
+        summary_header = additional_summary
+        EventMachine.stop
+      end
+      csv_file.close if csv_file
+      if @summary_output
+        File.open(@summary_output, "w") do |summary|
+          summary.puts summary_header
+          summary.puts "Log File: #{ARGV[0]}"
+          summary.puts "Speed: #{@speed}"
+          summary.puts "Concurrency: #{@concurrency}"
+          summary.puts "Average request time,#{total_time/count}"
+          summary.puts "Max request time,#{max_request_processing_time}"
+          summary.puts "Min request time,#{min_request_processing_time}"
+          summary.puts "Errors,#{error_count}"
+          status_counts.each { |status, count| summary.puts "Status #{status},#{count}" }
+        end
+      end
+    end
+
+    def self.execute
+      options = Trollop::options do
+        banner "Usage: #{$0} [options] sas_log_file"
+        opt :host, "Host to run load test against", :type => :string, :default => "localhost:4567"
+        opt :concurrency, "Client-side concurrency; maximum number of simultaneous requests", :type => :int,
+                          :default => 40
+        opt :speed, "Multiple of real-time to play back requests", :type => :float, :default => 1.0
+        opt :csv_output, "File to output CSV data to", :type => :string
+        opt :summary_output, "File to output summary data to", :type => :string
+        opt :force, "Overwrite existing output file"
+        opt :profiler, "trigger ruby prof"
+      end
+      if options[:csv_output] && File.exists?(options[:csv_output]) && !options[:force]
+        STDERR.puts "Cowardly refusing to overwrite existing file '#{options[:csv_output]}'.  " +
+                    "Use --force to provide extra bravery."
+        exit 1
+      end
+      Trollop::die "missing log file" unless ARGV[0]
+      load_tester = self.new(options)
+      load_tester.load_request_routes(ARGV[0])
+      load_tester.replay_requests
+    end
+  end
+
+
+end

data/log_replayer.gemspec

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Bo Chen"]
+  gem.email         = ["bochen@ooyala.com"]
+  gem.description   = %q{Reads access logs and replays them}
+  gem.summary       = %q{Use to replay load test by replaying logs at different speeds}
+  gem.homepage      = ""
+
+  gem.version       = "0.0.4"
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "log_replayer"
+  gem.require_paths = ["lib"]
+  gem.add_dependency "em-synchrony"
+  gem.add_dependency "em-http-request"
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: log_replayer
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+  prerelease: 
+platform: ruby
+authors:
+- Bo Chen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-15 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: &70096980187540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70096980187540
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &70096980187060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70096980187060
+description: Reads access logs and replays them
+email:
+- bochen@ooyala.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- lib/log_replayer.rb
+- log_replayer.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Use to replay load test by replaying logs at different speeds
+test_files: []