bigbench 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.textile +60 -22
- data/Rakefile +7 -0
- data/bigbench.gemspec +5 -0
- data/dev/net_http.rb +78 -0
- data/dev/test.rb +13 -0
- data/dev/tracking.rb +49 -0
- data/doc/BigBench.html +22 -17
- data/doc/BigBench/Benchmark.html +12 -8
- data/doc/BigBench/Benchmark/Benchmark.html +17 -27
- data/doc/BigBench/Benchmark/Looper.html +340 -0
- data/doc/BigBench/Bot.html +4 -2
- data/doc/BigBench/Configuration.html +20 -11
- data/doc/BigBench/Configuration/Config.html +23 -8
- data/doc/BigBench/Configuration/InvalidOptions.html +5 -3
- data/doc/BigBench/Executor.html +4 -2
- data/doc/BigBench/Executor/InvalidCommand.html +4 -2
- data/doc/BigBench/Fragment.html +46 -29
- data/doc/BigBench/Fragment/Fragment.html +18 -28
- data/doc/BigBench/Output.html +4 -2
- data/doc/BigBench/Runner.html +12 -9
- data/doc/BigBench/Runner/NoBenchmarksDefined.html +4 -2
- data/doc/BigBench/Store.html +4 -2
- data/doc/BigBench/Tracker.html +4 -2
- data/doc/BigBench/Tracker/Tracker.html +5 -3
- data/doc/EventMachineLoop.html +296 -0
- data/doc/Float.html +4 -2
- data/doc/Gemfile.html +4 -2
- data/doc/Helpers.html +4 -2
- data/doc/Object.html +87 -2
- data/doc/Rakefile.html +10 -3
- data/doc/created.rid +36 -31
- data/doc/index.html +4 -2
- data/doc/js/search_index.js +1 -1
- data/doc/lib/bigbench/help/executor_txt.html +4 -2
- data/doc/table_of_contents.html +31 -16
- data/lib/bigbench.rb +3 -0
- data/lib/bigbench/benchmark.rb +11 -21
- data/lib/bigbench/benchmark/looper.rb +43 -0
- data/lib/bigbench/configuration.rb +17 -10
- data/lib/bigbench/fragment.rb +47 -34
- data/lib/bigbench/runner.rb +8 -7
- data/lib/bigbench/tracker.rb +3 -3
- data/lib/bigbench/version.rb +1 -1
- data/spec/benchmark_spec.rb +14 -7
- data/spec/configure_spec.rb +3 -3
- data/spec/fragment_spec.rb +186 -24
- data/spec/helpers.rb +2 -4
- data/spec/lib/test_web_server.rb +44 -15
- data/spec/looper_spec.rb +47 -0
- data/spec/tests/local.rb +1 -1
- data/spec/webserver_spec.rb +91 -21
- metadata +77 -15
data/lib/bigbench.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require 'redis'
|
2
2
|
require 'net/http'
|
3
|
+
require 'eventmachine'
|
4
|
+
require 'em-http'
|
3
5
|
require 'active_support/secure_random'
|
4
6
|
require 'active_support/all'
|
5
7
|
|
@@ -7,6 +9,7 @@ require "bigbench/float_extensions"
|
|
7
9
|
require "bigbench/version"
|
8
10
|
require "bigbench/configuration"
|
9
11
|
require "bigbench/benchmark"
|
12
|
+
require 'bigbench/benchmark/looper'
|
10
13
|
require "bigbench/fragment"
|
11
14
|
require "bigbench/tracker"
|
12
15
|
require "bigbench/runner"
|
data/lib/bigbench/benchmark.rb
CHANGED
@@ -7,11 +7,12 @@ module BigBench
|
|
7
7
|
# get "/"
|
8
8
|
# get "/blog"
|
9
9
|
# get "/imprint"
|
10
|
+
# get "/admin", :basic_auth => ['admin', 'secret']
|
10
11
|
# end
|
11
12
|
#
|
12
13
|
# benchmark "login and logout" => "http://localhost:3000" do
|
13
|
-
# post "/login", { :name => "test@user.com", :password => "secret" }
|
14
|
-
# post "/logout"
|
14
|
+
# post "/login", :params => { :name => "test@user.com", :password => "secret" }
|
15
|
+
# post "/logout", :params => { :name => "test@user.com" }
|
15
16
|
# end
|
16
17
|
#
|
17
18
|
# Those benchmarks automatically get added to the modules benchmark array and can be retrieved with the BigBench#benchmarks method.
|
@@ -23,7 +24,7 @@ module BigBench
|
|
23
24
|
|
24
25
|
attr_accessor :name
|
25
26
|
attr_accessor :uri
|
26
|
-
attr_accessor :
|
27
|
+
attr_accessor :users
|
27
28
|
attr_accessor :duration
|
28
29
|
attr_accessor :fragments
|
29
30
|
attr_accessor :tracker
|
@@ -34,7 +35,7 @@ module BigBench
|
|
34
35
|
# Initizalizes a new benchmark
|
35
36
|
def initialize(name, url, options, &block)
|
36
37
|
@name, @uri, @tracker, @is_running, @runs = name, URI(url), Tracker::Tracker.new, false, 0
|
37
|
-
@
|
38
|
+
@users = options[:users] || BigBench.config.users
|
38
39
|
@duration = options[:duration] || BigBench.config.duration
|
39
40
|
@fragments = BigBench::Fragment.parse(self, &block)
|
40
41
|
end
|
@@ -42,24 +43,12 @@ module BigBench
|
|
42
43
|
# Execute this benchmark
|
43
44
|
def run!
|
44
45
|
|
45
|
-
# Setup
|
46
|
+
# Setup Timer
|
46
47
|
timer = Thread.new{ sleep(@duration); @is_running = false }
|
47
48
|
@start, @is_running = Time.now, true
|
48
49
|
|
49
|
-
#
|
50
|
-
@
|
51
|
-
@threads.times do
|
52
|
-
@running_threads << Thread.new do
|
53
|
-
while is_running? do
|
54
|
-
Net::HTTP.start(uri.host, uri.port) do |http|
|
55
|
-
@fragments.each{ |fragment| fragment.run!(http) }
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
# Stop execution
|
62
|
-
@running_threads.each { |thread| thread.join }
|
50
|
+
# Loop Requests
|
51
|
+
@users.times { Looper.new(self).loop! }
|
63
52
|
@runs += 1
|
64
53
|
end
|
65
54
|
|
@@ -101,11 +90,12 @@ module BigBench
|
|
101
90
|
# get "/"
|
102
91
|
# get "/blog"
|
103
92
|
# get "/imprint"
|
93
|
+
# get "/admin", :basic_auth => ['admin', 'secret']
|
104
94
|
# end
|
105
95
|
#
|
106
96
|
# benchmark "login and logout" => "http://localhost:3000" do
|
107
|
-
# post "/login", { :name => "test@user.com", :password => "secret" }
|
108
|
-
# post "
|
97
|
+
# post "/login", :params => { :name => "test@user.com", :password => "secret" }
|
98
|
+
# post "/logout", :params => { :name => "test@user.com" }
|
109
99
|
# end
|
110
100
|
#
|
111
101
|
def self.benchmark(options)
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module BigBench
|
2
|
+
module Benchmark
|
3
|
+
|
4
|
+
class Looper
|
5
|
+
|
6
|
+
attr_accessor :benchmark
|
7
|
+
attr_accessor :loops
|
8
|
+
|
9
|
+
def initialize(benchmark)
|
10
|
+
@benchmark, @loops = benchmark, 0
|
11
|
+
@fragments = Fiber.new { @benchmark.fragments.cycle{ |fragment| Fiber.yield(fragment) } }
|
12
|
+
end
|
13
|
+
|
14
|
+
def loop!
|
15
|
+
return unless @benchmark.is_running?
|
16
|
+
@loops += 1
|
17
|
+
|
18
|
+
# Start next fragment
|
19
|
+
fragment = next_fragment
|
20
|
+
start = Time.now
|
21
|
+
http = fragment.run!
|
22
|
+
|
23
|
+
# Success
|
24
|
+
http.callback do |http|
|
25
|
+
fragment.track!(start, Time.now, http)
|
26
|
+
loop!
|
27
|
+
end
|
28
|
+
|
29
|
+
# Error
|
30
|
+
http.errback do |http|
|
31
|
+
fragment.track!(start, Time.now, http)
|
32
|
+
loop!
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
def next_fragment
|
38
|
+
@fragments.resume
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -3,9 +3,10 @@ module BigBench
|
|
3
3
|
# The configuration is configured in the test reciepts and looks like this:
|
4
4
|
#
|
5
5
|
# BigBench.configure = {
|
6
|
-
# :duration
|
7
|
-
# :output
|
8
|
-
# :
|
6
|
+
# :duration => 10.seconds,
|
7
|
+
# :output => "test.ljson",
|
8
|
+
# :users => 20,
|
9
|
+
# :basic_auth => ['username', 'secret']
|
9
10
|
# }
|
10
11
|
#
|
11
12
|
# Single values can be set and retrieved like this:
|
@@ -19,7 +20,11 @@ module BigBench
|
|
19
20
|
# [output*] The file where the results should be written to. Usually one takes the <tt>*.ljson</tt> format which is simply a file
|
20
21
|
# that contains a fully valid JSON object on every line. This makes it easy to process the results later on, because there's
|
21
22
|
# no need to parse the whole results JSON, which can get huge, at once.
|
22
|
-
# [
|
23
|
+
# [users] How many users execute the benchmarks concurrently. This can be overridden by every benchmark and defaults to <tt>1</tt>.
|
24
|
+
# [basic_auth] A basic authentication that is used by ALL requests. This can be overridden per fragment in every benchmark.
|
25
|
+
#
|
26
|
+
# BigBench.config.basic_auth = ['username', 'password']
|
27
|
+
#
|
23
28
|
module Configuration
|
24
29
|
|
25
30
|
# The main config object for BigBench. It allows config options to
|
@@ -27,14 +32,15 @@ module BigBench
|
|
27
32
|
class Config
|
28
33
|
attr_accessor :duration
|
29
34
|
attr_accessor :output
|
30
|
-
attr_accessor :
|
35
|
+
attr_accessor :users
|
31
36
|
attr_accessor :mode
|
32
37
|
attr_accessor :bot_checks_every
|
38
|
+
attr_accessor :basic_auth
|
33
39
|
|
34
|
-
VALIDATE_OPTIONS = [:duration, :output, :
|
40
|
+
VALIDATE_OPTIONS = [:duration, :output, :users]
|
35
41
|
|
36
42
|
def initialize
|
37
|
-
@
|
43
|
+
@users, @duration, @mode, @bot_checks_every = 1, 1.second, :local, 1.minute
|
38
44
|
end
|
39
45
|
|
40
46
|
def self.add_option(name)
|
@@ -81,9 +87,10 @@ module BigBench
|
|
81
87
|
# Configure the benchmark by supplying a hash of options like this:
|
82
88
|
#
|
83
89
|
# BigBench.configure = {
|
84
|
-
# :duration
|
85
|
-
# :output
|
86
|
-
# :
|
90
|
+
# :duration => 10.seconds,
|
91
|
+
# :output => "test.ljson",
|
92
|
+
# :users => 20,
|
93
|
+
# :basic_auth => ['username', 'secret']
|
87
94
|
# }
|
88
95
|
#
|
89
96
|
# Those values can then be set and retreived with <tt>BigBench.config.duration</tt>, ...
|
data/lib/bigbench/fragment.rb
CHANGED
@@ -10,10 +10,22 @@ module BigBench
|
|
10
10
|
# Possible fragment types are the HTTP verbs, like GET, POST, PUT and DELETE. They look like this:
|
11
11
|
#
|
12
12
|
# get "/"
|
13
|
-
# post "/login/new"
|
14
|
-
# put "/books"
|
15
|
-
# delete "/books/5"
|
13
|
+
# post "/login/new"
|
14
|
+
# put "/books"
|
15
|
+
# delete "/books/5", :params => { :token => '87bas67dfjgbrjbbgbi6ica7s0b3t0' }
|
16
|
+
# get "/admin", :basic_auth => ['username', 'password']
|
16
17
|
#
|
18
|
+
# After any fragment an options hash can be appended. Possible options are:
|
19
|
+
#
|
20
|
+
# [:basic_auth] Supply a basic auth user and password for the request. The request then looks like this:
|
21
|
+
#
|
22
|
+
# get "/", :basic_auth => ['username', 'password']
|
23
|
+
#
|
24
|
+
# [:params] Supply params for the request body, like e.g. form data:
|
25
|
+
#
|
26
|
+
# post "/books/new", :params => { :book => { :title => "Metaprogramming Ruby", :publisher => "O'Reilly" } }
|
27
|
+
# post "/trackings/add", :params => { :name => "Tommy" }
|
28
|
+
#
|
17
29
|
module Fragment
|
18
30
|
|
19
31
|
@fragments = []
|
@@ -24,34 +36,22 @@ module BigBench
|
|
24
36
|
attr_accessor :benchmark
|
25
37
|
attr_accessor :path
|
26
38
|
attr_accessor :method
|
27
|
-
attr_accessor :
|
39
|
+
attr_accessor :options
|
28
40
|
attr_accessor :uri
|
29
41
|
|
30
|
-
def initialize benchmark, path, method,
|
31
|
-
@benchmark, @path, @method, @
|
42
|
+
def initialize benchmark, path, method, options = {}
|
43
|
+
@benchmark, @path, @method, @options, @request_options = benchmark, path, method, options, {}
|
32
44
|
@uri = URI(@benchmark.uri.to_s + @path)
|
45
|
+
configure_options
|
33
46
|
end
|
34
47
|
|
35
48
|
# Initiates the request in the context of a <tt>Net::HTTP.start</tt> block
|
36
|
-
def run!
|
37
|
-
|
38
|
-
when :get then Net::HTTP::Get.new(@uri.request_uri)
|
39
|
-
when :post then Net::HTTP::Post.new(@uri.request_uri)
|
40
|
-
when :put then Net::HTTP::Put.new(@uri.request_uri)
|
41
|
-
when :delete then Net::HTTP::Delete.new(@uri.request_uri)
|
42
|
-
else nil
|
43
|
-
end
|
44
|
-
|
45
|
-
start = Time.now
|
46
|
-
response = http.request(request)
|
47
|
-
stop = Time.now
|
48
|
-
|
49
|
-
track!(start, stop, response)
|
50
|
-
response
|
49
|
+
def run!
|
50
|
+
EventMachine::HttpRequest.new(@uri.to_s).send(@method, @request_options)
|
51
51
|
end
|
52
52
|
|
53
53
|
# Adds the current tracking result as a hash to the benchmark's tracker
|
54
|
-
def track!(start, stop,
|
54
|
+
def track!(start, stop, http)
|
55
55
|
@benchmark.tracker.track(
|
56
56
|
{
|
57
57
|
:elapsed => (stop - benchmark.start).to_f,
|
@@ -62,42 +62,55 @@ module BigBench
|
|
62
62
|
:url => @uri.to_s,
|
63
63
|
:path => @uri.request_uri,
|
64
64
|
:method => @method,
|
65
|
-
:status =>
|
65
|
+
:status => http.response_header.status
|
66
66
|
}
|
67
67
|
)
|
68
68
|
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def configure_options
|
73
|
+
|
74
|
+
# Basic Auth
|
75
|
+
@request_options[:head] = { 'authorization' => BigBench.config.basic_auth } unless BigBench.config.basic_auth.nil?
|
76
|
+
@request_options[:head] = { 'authorization' => @options[:basic_auth] } unless @options[:basic_auth].nil?
|
77
|
+
|
78
|
+
# Body
|
79
|
+
@request_options[:body] = @options[:params] unless @options[:params].nil?
|
80
|
+
|
81
|
+
end
|
69
82
|
end
|
70
83
|
|
71
84
|
# Performs a GET request to the given url, e.g.
|
72
85
|
#
|
73
86
|
# get "/some/page"
|
74
87
|
#
|
75
|
-
def self.get(path)
|
76
|
-
@fragments << Fragment.new(@benchmark, path, :get,
|
88
|
+
def self.get(path, options = {})
|
89
|
+
@fragments << Fragment.new(@benchmark, path, :get, options)
|
77
90
|
end
|
78
91
|
|
79
92
|
# Performs a POST request to the given url, e.g.
|
80
93
|
#
|
81
|
-
# post "/login"
|
94
|
+
# post "/login"
|
82
95
|
#
|
83
|
-
def self.post(path,
|
84
|
-
@fragments << Fragment.new(@benchmark, path, :post,
|
96
|
+
def self.post(path, options = {})
|
97
|
+
@fragments << Fragment.new(@benchmark, path, :post, options)
|
85
98
|
end
|
86
99
|
|
87
100
|
# Performs a PUT request to the given url, e.g.
|
88
101
|
#
|
89
|
-
# put "/books"
|
102
|
+
# put "/books"
|
90
103
|
#
|
91
|
-
def self.put(path,
|
92
|
-
@fragments << Fragment.new(@benchmark, path, :put,
|
104
|
+
def self.put(path, options = {})
|
105
|
+
@fragments << Fragment.new(@benchmark, path, :put, options)
|
93
106
|
end
|
94
107
|
|
95
108
|
# Performs a DELETE request to the given url, e.g.
|
96
109
|
#
|
97
|
-
# delete "/books/5"
|
110
|
+
# delete "/books/5"
|
98
111
|
#
|
99
|
-
def self.delete(path,
|
100
|
-
@fragments << Fragment.new(@benchmark, path, :delete,
|
112
|
+
def self.delete(path, options = {})
|
113
|
+
@fragments << Fragment.new(@benchmark, path, :delete, options)
|
101
114
|
end
|
102
115
|
|
103
116
|
# Evaluates a benchmark block full of puts and gets and returns an array of fully configured fragments for it
|
data/lib/bigbench/runner.rb
CHANGED
@@ -16,13 +16,14 @@ module BigBench
|
|
16
16
|
raise NoBenchmarksDefined.new if BigBench.benchmarks.empty?
|
17
17
|
|
18
18
|
# Run all benchmarks
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
19
|
+
EventMachine.run {
|
20
|
+
|
21
|
+
# Start Timer
|
22
|
+
Thread.new { sleep(BigBench.duration.to_i); EventMachine.stop }
|
23
|
+
|
24
|
+
# Start Benchmarks
|
25
|
+
BigBench.benchmarks.each{ |benchmark| benchmark.run! }
|
26
|
+
}
|
26
27
|
end
|
27
28
|
|
28
29
|
end
|
data/lib/bigbench/tracker.rb
CHANGED
@@ -18,7 +18,7 @@ module BigBench
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def track object
|
21
|
-
@trackings << object
|
21
|
+
@trackings << object
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
@@ -33,7 +33,7 @@ module BigBench
|
|
33
33
|
BigBench.benchmarks.each do |benchmark|
|
34
34
|
benchmark.tracker.trackings.each do |tracking|
|
35
35
|
BigBench::Output.wrote_trackings(counter) if counter % 100 == 0
|
36
|
-
BigBench::Store.add_tracking(tracking)
|
36
|
+
BigBench::Store.add_tracking(tracking.to_json)
|
37
37
|
counter += 1
|
38
38
|
end
|
39
39
|
end
|
@@ -51,7 +51,7 @@ module BigBench
|
|
51
51
|
BigBench.benchmarks.each do |benchmark|
|
52
52
|
benchmark.tracker.trackings.each do |tracking|
|
53
53
|
BigBench::Output.wrote_trackings(counter) if counter % 100 == 0
|
54
|
-
file.write tracking + "\n"
|
54
|
+
file.write tracking.to_json + "\n"
|
55
55
|
counter += 1
|
56
56
|
end
|
57
57
|
end
|
data/lib/bigbench/version.rb
CHANGED
data/spec/benchmark_spec.rb
CHANGED
@@ -15,18 +15,18 @@ describe BigBench::Benchmark do
|
|
15
15
|
|
16
16
|
BigBench.benchmarks.size.should == 1
|
17
17
|
BigBench.benchmarks.first.fragments.size.should == 1
|
18
|
-
BigBench.benchmarks.first.
|
18
|
+
BigBench.benchmarks.first.users.should == 1
|
19
19
|
BigBench.benchmarks.first.fragments.first.uri.to_s.should == "http://localhost:3001/"
|
20
20
|
end
|
21
21
|
|
22
|
-
it "should add a benchmark with custom
|
23
|
-
BigBench.benchmark "a test page" => "http://localhost:3001", :
|
22
|
+
it "should add a benchmark with custom users" do
|
23
|
+
BigBench.benchmark "a test page" => "http://localhost:3001", :users => 20 do
|
24
24
|
get "/"
|
25
25
|
end
|
26
26
|
|
27
27
|
BigBench.benchmarks.size.should == 1
|
28
28
|
BigBench.benchmarks.first.fragments.size.should == 1
|
29
|
-
BigBench.benchmarks.first.
|
29
|
+
BigBench.benchmarks.first.users.should == 20
|
30
30
|
end
|
31
31
|
|
32
32
|
it "should return the longest duration of all benchmarks" do
|
@@ -49,17 +49,24 @@ describe BigBench::Benchmark do
|
|
49
49
|
|
50
50
|
it "make 3 GETs" do
|
51
51
|
|
52
|
-
benchmark = BigBench.benchmark "a test page" => "http://localhost:3001" do
|
52
|
+
benchmark = BigBench.benchmark "a test page" => "http://localhost:3001", :users => 5 do
|
53
53
|
get "/"
|
54
54
|
get "/index"
|
55
55
|
get "/blog"
|
56
56
|
end
|
57
57
|
|
58
|
-
|
58
|
+
EventMachine.run{
|
59
|
+
Thread.new{ sleep(2); EventMachine.stop }
|
60
|
+
benchmark.run!
|
61
|
+
}
|
62
|
+
|
59
63
|
benchmark.tracker.trackings.size.should > 100
|
60
64
|
benchmark.runs.should == 1
|
61
65
|
|
62
|
-
|
66
|
+
EventMachine.run{
|
67
|
+
Thread.new{ sleep(2); EventMachine.stop }
|
68
|
+
benchmark.run!
|
69
|
+
}
|
63
70
|
benchmark.tracker.trackings.size.should > 100
|
64
71
|
benchmark.runs.should == 2
|
65
72
|
end
|
data/spec/configure_spec.rb
CHANGED
@@ -7,12 +7,12 @@ describe BigBench::Configuration do
|
|
7
7
|
BigBench.configure = {
|
8
8
|
:duration => 10.seconds,
|
9
9
|
:output => "test.ljson",
|
10
|
-
:
|
10
|
+
:users => 20
|
11
11
|
}
|
12
12
|
|
13
13
|
BigBench.config.duration.should == 10.seconds
|
14
14
|
BigBench.config.output.should == "test.ljson"
|
15
|
-
BigBench.config.
|
15
|
+
BigBench.config.users.should == 20
|
16
16
|
|
17
17
|
end
|
18
18
|
|
@@ -27,7 +27,7 @@ describe BigBench::Configuration do
|
|
27
27
|
BigBench.configure = {
|
28
28
|
:duration => 10.seconds,
|
29
29
|
:output => "test.ljson",
|
30
|
-
:
|
30
|
+
:users => 20
|
31
31
|
}
|
32
32
|
BigBench.config.should be_valid
|
33
33
|
end
|