bigbench 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/README.textile +60 -22
  2. data/Rakefile +7 -0
  3. data/bigbench.gemspec +5 -0
  4. data/dev/net_http.rb +78 -0
  5. data/dev/test.rb +13 -0
  6. data/dev/tracking.rb +49 -0
  7. data/doc/BigBench.html +22 -17
  8. data/doc/BigBench/Benchmark.html +12 -8
  9. data/doc/BigBench/Benchmark/Benchmark.html +17 -27
  10. data/doc/BigBench/Benchmark/Looper.html +340 -0
  11. data/doc/BigBench/Bot.html +4 -2
  12. data/doc/BigBench/Configuration.html +20 -11
  13. data/doc/BigBench/Configuration/Config.html +23 -8
  14. data/doc/BigBench/Configuration/InvalidOptions.html +5 -3
  15. data/doc/BigBench/Executor.html +4 -2
  16. data/doc/BigBench/Executor/InvalidCommand.html +4 -2
  17. data/doc/BigBench/Fragment.html +46 -29
  18. data/doc/BigBench/Fragment/Fragment.html +18 -28
  19. data/doc/BigBench/Output.html +4 -2
  20. data/doc/BigBench/Runner.html +12 -9
  21. data/doc/BigBench/Runner/NoBenchmarksDefined.html +4 -2
  22. data/doc/BigBench/Store.html +4 -2
  23. data/doc/BigBench/Tracker.html +4 -2
  24. data/doc/BigBench/Tracker/Tracker.html +5 -3
  25. data/doc/EventMachineLoop.html +296 -0
  26. data/doc/Float.html +4 -2
  27. data/doc/Gemfile.html +4 -2
  28. data/doc/Helpers.html +4 -2
  29. data/doc/Object.html +87 -2
  30. data/doc/Rakefile.html +10 -3
  31. data/doc/created.rid +36 -31
  32. data/doc/index.html +4 -2
  33. data/doc/js/search_index.js +1 -1
  34. data/doc/lib/bigbench/help/executor_txt.html +4 -2
  35. data/doc/table_of_contents.html +31 -16
  36. data/lib/bigbench.rb +3 -0
  37. data/lib/bigbench/benchmark.rb +11 -21
  38. data/lib/bigbench/benchmark/looper.rb +43 -0
  39. data/lib/bigbench/configuration.rb +17 -10
  40. data/lib/bigbench/fragment.rb +47 -34
  41. data/lib/bigbench/runner.rb +8 -7
  42. data/lib/bigbench/tracker.rb +3 -3
  43. data/lib/bigbench/version.rb +1 -1
  44. data/spec/benchmark_spec.rb +14 -7
  45. data/spec/configure_spec.rb +3 -3
  46. data/spec/fragment_spec.rb +186 -24
  47. data/spec/helpers.rb +2 -4
  48. data/spec/lib/test_web_server.rb +44 -15
  49. data/spec/looper_spec.rb +47 -0
  50. data/spec/tests/local.rb +1 -1
  51. data/spec/webserver_spec.rb +91 -21
  52. metadata +77 -15
@@ -1,5 +1,7 @@
1
1
  require 'redis'
2
2
  require 'net/http'
3
+ require 'eventmachine'
4
+ require 'em-http'
3
5
  require 'active_support/secure_random'
4
6
  require 'active_support/all'
5
7
 
@@ -7,6 +9,7 @@ require "bigbench/float_extensions"
7
9
  require "bigbench/version"
8
10
  require "bigbench/configuration"
9
11
  require "bigbench/benchmark"
12
+ require 'bigbench/benchmark/looper'
10
13
  require "bigbench/fragment"
11
14
  require "bigbench/tracker"
12
15
  require "bigbench/runner"
@@ -7,11 +7,12 @@ module BigBench
7
7
  # get "/"
8
8
  # get "/blog"
9
9
  # get "/imprint"
10
+ # get "/admin", :basic_auth => ['admin', 'secret']
10
11
  # end
11
12
  #
12
13
  # benchmark "login and logout" => "http://localhost:3000" do
13
- # post "/login", { :name => "test@user.com", :password => "secret" }
14
- # post "/logout"
14
+ # post "/login", :params => { :name => "test@user.com", :password => "secret" }
15
+ # post "/logout", :params => { :name => "test@user.com" }
15
16
  # end
16
17
  #
17
18
  # Those benchmarks automatically get added to the modules benchmark array and can be retrieved with the BigBench#benchmarks method.
@@ -23,7 +24,7 @@ module BigBench
23
24
 
24
25
  attr_accessor :name
25
26
  attr_accessor :uri
26
- attr_accessor :threads
27
+ attr_accessor :users
27
28
  attr_accessor :duration
28
29
  attr_accessor :fragments
29
30
  attr_accessor :tracker
@@ -34,7 +35,7 @@ module BigBench
34
35
  # Initizalizes a new benchmark
35
36
  def initialize(name, url, options, &block)
36
37
  @name, @uri, @tracker, @is_running, @runs = name, URI(url), Tracker::Tracker.new, false, 0
37
- @threads = options[:threads] || BigBench.config.threads
38
+ @users = options[:users] || BigBench.config.users
38
39
  @duration = options[:duration] || BigBench.config.duration
39
40
  @fragments = BigBench::Fragment.parse(self, &block)
40
41
  end
@@ -42,24 +43,12 @@ module BigBench
42
43
  # Execute this benchmark
43
44
  def run!
44
45
 
45
- # Setup timer
46
+ # Setup Timer
46
47
  timer = Thread.new{ sleep(@duration); @is_running = false }
47
48
  @start, @is_running = Time.now, true
48
49
 
49
- # Benchmark loop
50
- @running_threads = []
51
- @threads.times do
52
- @running_threads << Thread.new do
53
- while is_running? do
54
- Net::HTTP.start(uri.host, uri.port) do |http|
55
- @fragments.each{ |fragment| fragment.run!(http) }
56
- end
57
- end
58
- end
59
- end
60
-
61
- # Stop execution
62
- @running_threads.each { |thread| thread.join }
50
+ # Loop Requests
51
+ @users.times { Looper.new(self).loop! }
63
52
  @runs += 1
64
53
  end
65
54
 
@@ -101,11 +90,12 @@ module BigBench
101
90
  # get "/"
102
91
  # get "/blog"
103
92
  # get "/imprint"
93
+ # get "/admin", :basic_auth => ['admin', 'secret']
104
94
  # end
105
95
  #
106
96
  # benchmark "login and logout" => "http://localhost:3000" do
107
- # post "/login", { :name => "test@user.com", :password => "secret" }
108
- # post "http://localhost:3000/logout"
97
+ # post "/login", :params => { :name => "test@user.com", :password => "secret" }
98
+ # post "/logout", :params => { :name => "test@user.com" }
109
99
  # end
110
100
  #
111
101
  def self.benchmark(options)
@@ -0,0 +1,43 @@
1
+ module BigBench
2
+ module Benchmark
3
+
4
+ class Looper
5
+
6
+ attr_accessor :benchmark
7
+ attr_accessor :loops
8
+
9
+ def initialize(benchmark)
10
+ @benchmark, @loops = benchmark, 0
11
+ @fragments = Fiber.new { @benchmark.fragments.cycle{ |fragment| Fiber.yield(fragment) } }
12
+ end
13
+
14
+ def loop!
15
+ return unless @benchmark.is_running?
16
+ @loops += 1
17
+
18
+ # Start next fragment
19
+ fragment = next_fragment
20
+ start = Time.now
21
+ http = fragment.run!
22
+
23
+ # Success
24
+ http.callback do |http|
25
+ fragment.track!(start, Time.now, http)
26
+ loop!
27
+ end
28
+
29
+ # Error
30
+ http.errback do |http|
31
+ fragment.track!(start, Time.now, http)
32
+ loop!
33
+ end
34
+
35
+ end
36
+
37
+ def next_fragment
38
+ @fragments.resume
39
+ end
40
+
41
+ end
42
+ end
43
+ end
@@ -3,9 +3,10 @@ module BigBench
3
3
  # The configuration is configured in the test reciepts and looks like this:
4
4
  #
5
5
  # BigBench.configure = {
6
- # :duration => 10.seconds,
7
- # :output => "test.ljson",
8
- # :threads => 20
6
+ # :duration => 10.seconds,
7
+ # :output => "test.ljson",
8
+ # :users => 20,
9
+ # :basic_auth => ['username', 'secret']
9
10
  # }
10
11
  #
11
12
  # Single values can be set and retrieved like this:
@@ -19,7 +20,11 @@ module BigBench
19
20
  # [output*] The file where the results should be written to. Usually one takes the <tt>*.ljson</tt> format which is simply a file
20
21
  # that contains a fully valid JSON object on every line. This makes it easy to process the results later on, because there's
21
22
  # no need to parse the whole results JSON, which can get huge, at once.
22
- # [threads] How many threads execute the benchmarks. This can be overridden by every benchmark and defaults to <tt>1</tt>.
23
+ # [users] How many users execute the benchmarks concurrently. This can be overridden by every benchmark and defaults to <tt>1</tt>.
24
+ # [basic_auth] A basic authentication that is used by ALL requests. This can be overridden per fragment in every benchmark.
25
+ #
26
+ # BigBench.config.basic_auth = ['username', 'password']
27
+ #
23
28
  module Configuration
24
29
 
25
30
  # The main config object for BigBench. It allows config options to
@@ -27,14 +32,15 @@ module BigBench
27
32
  class Config
28
33
  attr_accessor :duration
29
34
  attr_accessor :output
30
- attr_accessor :threads
35
+ attr_accessor :users
31
36
  attr_accessor :mode
32
37
  attr_accessor :bot_checks_every
38
+ attr_accessor :basic_auth
33
39
 
34
- VALIDATE_OPTIONS = [:duration, :output, :threads]
40
+ VALIDATE_OPTIONS = [:duration, :output, :users]
35
41
 
36
42
  def initialize
37
- @threads, @duration, @mode, @bot_checks_every = 1, 1.second, :local, 1.minute
43
+ @users, @duration, @mode, @bot_checks_every = 1, 1.second, :local, 1.minute
38
44
  end
39
45
 
40
46
  def self.add_option(name)
@@ -81,9 +87,10 @@ module BigBench
81
87
  # Configure the benchmark by supplying a hash of options like this:
82
88
  #
83
89
  # BigBench.configure = {
84
- # :duration => 10.seconds,
85
- # :output => "test.ljson",
86
- # :threads => 20
90
+ # :duration => 10.seconds,
91
+ # :output => "test.ljson",
92
+ # :users => 20,
93
+ # :basic_auth => ['username', 'secret']
87
94
  # }
88
95
  #
89
96
  # Those values can then be set and retreived with <tt>BigBench.config.duration</tt>, ...
@@ -10,10 +10,22 @@ module BigBench
10
10
  # Possible fragment types are the HTTP verbs, like GET, POST, PUT and DELETE. They look like this:
11
11
  #
12
12
  # get "/"
13
- # post "/login/new", { :name => "a name", :password => "secret" }
14
- # put "/books", { :name => "A Book title", :rating => 30.4 }
15
- # delete "/books/5"
13
+ # post "/login/new"
14
+ # put "/books"
15
+ # delete "/books/5", :params => { :token => '87bas67dfjgbrjbbgbi6ica7s0b3t0' }
16
+ # get "/admin", :basic_auth => ['username', 'password']
16
17
  #
18
+ # After any fragment an options hash can be appended. Possible options are:
19
+ #
20
+ # [:basic_auth] Supply a basic auth user and password for the request. The request then looks like this:
21
+ #
22
+ # get "/", :basic_auth => ['username', 'password']
23
+ #
24
+ # [:params] Supply params for the request body, like e.g. form data:
25
+ #
26
+ # post "/books/new", :params => { :book => { :title => "Metaprogramming Ruby", :publisher => "O'Reilly" } }
27
+ # post "/trackings/add", :params => { :name => "Tommy" }
28
+ #
17
29
  module Fragment
18
30
 
19
31
  @fragments = []
@@ -24,34 +36,22 @@ module BigBench
24
36
  attr_accessor :benchmark
25
37
  attr_accessor :path
26
38
  attr_accessor :method
27
- attr_accessor :params
39
+ attr_accessor :options
28
40
  attr_accessor :uri
29
41
 
30
- def initialize benchmark, path, method, params = {}
31
- @benchmark, @path, @method, @params = benchmark, path, method, params
42
+ def initialize benchmark, path, method, options = {}
43
+ @benchmark, @path, @method, @options, @request_options = benchmark, path, method, options, {}
32
44
  @uri = URI(@benchmark.uri.to_s + @path)
45
+ configure_options
33
46
  end
34
47
 
35
48
  # Initiates the request in the context of a <tt>Net::HTTP.start</tt> block
36
- def run!(http)
37
- request = case @method
38
- when :get then Net::HTTP::Get.new(@uri.request_uri)
39
- when :post then Net::HTTP::Post.new(@uri.request_uri)
40
- when :put then Net::HTTP::Put.new(@uri.request_uri)
41
- when :delete then Net::HTTP::Delete.new(@uri.request_uri)
42
- else nil
43
- end
44
-
45
- start = Time.now
46
- response = http.request(request)
47
- stop = Time.now
48
-
49
- track!(start, stop, response)
50
- response
49
+ def run!
50
+ EventMachine::HttpRequest.new(@uri.to_s).send(@method, @request_options)
51
51
  end
52
52
 
53
53
  # Adds the current tracking result as a hash to the benchmark's tracker
54
- def track!(start, stop, response)
54
+ def track!(start, stop, http)
55
55
  @benchmark.tracker.track(
56
56
  {
57
57
  :elapsed => (stop - benchmark.start).to_f,
@@ -62,42 +62,55 @@ module BigBench
62
62
  :url => @uri.to_s,
63
63
  :path => @uri.request_uri,
64
64
  :method => @method,
65
- :status => response.code
65
+ :status => http.response_header.status
66
66
  }
67
67
  )
68
68
  end
69
+
70
+ private
71
+
72
+ def configure_options
73
+
74
+ # Basic Auth
75
+ @request_options[:head] = { 'authorization' => BigBench.config.basic_auth } unless BigBench.config.basic_auth.nil?
76
+ @request_options[:head] = { 'authorization' => @options[:basic_auth] } unless @options[:basic_auth].nil?
77
+
78
+ # Body
79
+ @request_options[:body] = @options[:params] unless @options[:params].nil?
80
+
81
+ end
69
82
  end
70
83
 
71
84
  # Performs a GET request to the given url, e.g.
72
85
  #
73
86
  # get "/some/page"
74
87
  #
75
- def self.get(path)
76
- @fragments << Fragment.new(@benchmark, path, :get, {})
88
+ def self.get(path, options = {})
89
+ @fragments << Fragment.new(@benchmark, path, :get, options)
77
90
  end
78
91
 
79
92
  # Performs a POST request to the given url, e.g.
80
93
  #
81
- # post "/login", { :user => "some@sample.com", :password => "secret" }
94
+ # post "/login"
82
95
  #
83
- def self.post(path, params = {})
84
- @fragments << Fragment.new(@benchmark, path, :post, params)
96
+ def self.post(path, options = {})
97
+ @fragments << Fragment.new(@benchmark, path, :post, options)
85
98
  end
86
99
 
87
100
  # Performs a PUT request to the given url, e.g.
88
101
  #
89
- # put "/books", { :book => "Some book content" }
102
+ # put "/books"
90
103
  #
91
- def self.put(path, params = {})
92
- @fragments << Fragment.new(@benchmark, path, :put, params)
104
+ def self.put(path, options = {})
105
+ @fragments << Fragment.new(@benchmark, path, :put, options)
93
106
  end
94
107
 
95
108
  # Performs a DELETE request to the given url, e.g.
96
109
  #
97
- # delete "/books/5", { :user => "some@sample.com", :password => "secret" }
110
+ # delete "/books/5"
98
111
  #
99
- def self.delete(path, params = {})
100
- @fragments << Fragment.new(@benchmark, path, :delete, params)
112
+ def self.delete(path, options = {})
113
+ @fragments << Fragment.new(@benchmark, path, :delete, options)
101
114
  end
102
115
 
103
116
  # Evaluates a benchmark block full of puts and gets and returns an array of fully configured fragments for it
@@ -16,13 +16,14 @@ module BigBench
16
16
  raise NoBenchmarksDefined.new if BigBench.benchmarks.empty?
17
17
 
18
18
  # Run all benchmarks
19
- @running_threads = []
20
- BigBench.benchmarks.each do |benchmark|
21
- @running_threads << Thread.new{ benchmark.run! }
22
- end
23
-
24
- # Wait for them to finish
25
- @running_threads.each{ |thread| thread.join }
19
+ EventMachine.run {
20
+
21
+ # Start Timer
22
+ Thread.new { sleep(BigBench.duration.to_i); EventMachine.stop }
23
+
24
+ # Start Benchmarks
25
+ BigBench.benchmarks.each{ |benchmark| benchmark.run! }
26
+ }
26
27
  end
27
28
 
28
29
  end
@@ -18,7 +18,7 @@ module BigBench
18
18
  end
19
19
 
20
20
  def track object
21
- @trackings << object.to_json
21
+ @trackings << object
22
22
  end
23
23
  end
24
24
 
@@ -33,7 +33,7 @@ module BigBench
33
33
  BigBench.benchmarks.each do |benchmark|
34
34
  benchmark.tracker.trackings.each do |tracking|
35
35
  BigBench::Output.wrote_trackings(counter) if counter % 100 == 0
36
- BigBench::Store.add_tracking(tracking)
36
+ BigBench::Store.add_tracking(tracking.to_json)
37
37
  counter += 1
38
38
  end
39
39
  end
@@ -51,7 +51,7 @@ module BigBench
51
51
  BigBench.benchmarks.each do |benchmark|
52
52
  benchmark.tracker.trackings.each do |tracking|
53
53
  BigBench::Output.wrote_trackings(counter) if counter % 100 == 0
54
- file.write tracking + "\n"
54
+ file.write tracking.to_json + "\n"
55
55
  counter += 1
56
56
  end
57
57
  end
@@ -1,3 +1,3 @@
1
1
  module BigBench
2
- VERSION = "0.0.1"
2
+ VERSION = "0.0.2"
3
3
  end
@@ -15,18 +15,18 @@ describe BigBench::Benchmark do
15
15
 
16
16
  BigBench.benchmarks.size.should == 1
17
17
  BigBench.benchmarks.first.fragments.size.should == 1
18
- BigBench.benchmarks.first.threads.should == 1
18
+ BigBench.benchmarks.first.users.should == 1
19
19
  BigBench.benchmarks.first.fragments.first.uri.to_s.should == "http://localhost:3001/"
20
20
  end
21
21
 
22
- it "should add a benchmark with custom threads" do
23
- BigBench.benchmark "a test page" => "http://localhost:3001", :threads => 20 do
22
+ it "should add a benchmark with custom users" do
23
+ BigBench.benchmark "a test page" => "http://localhost:3001", :users => 20 do
24
24
  get "/"
25
25
  end
26
26
 
27
27
  BigBench.benchmarks.size.should == 1
28
28
  BigBench.benchmarks.first.fragments.size.should == 1
29
- BigBench.benchmarks.first.threads.should == 20
29
+ BigBench.benchmarks.first.users.should == 20
30
30
  end
31
31
 
32
32
  it "should return the longest duration of all benchmarks" do
@@ -49,17 +49,24 @@ describe BigBench::Benchmark do
49
49
 
50
50
  it "make 3 GETs" do
51
51
 
52
- benchmark = BigBench.benchmark "a test page" => "http://localhost:3001" do
52
+ benchmark = BigBench.benchmark "a test page" => "http://localhost:3001", :users => 5 do
53
53
  get "/"
54
54
  get "/index"
55
55
  get "/blog"
56
56
  end
57
57
 
58
- benchmark.run!
58
+ EventMachine.run{
59
+ Thread.new{ sleep(2); EventMachine.stop }
60
+ benchmark.run!
61
+ }
62
+
59
63
  benchmark.tracker.trackings.size.should > 100
60
64
  benchmark.runs.should == 1
61
65
 
62
- benchmark.run!
66
+ EventMachine.run{
67
+ Thread.new{ sleep(2); EventMachine.stop }
68
+ benchmark.run!
69
+ }
63
70
  benchmark.tracker.trackings.size.should > 100
64
71
  benchmark.runs.should == 2
65
72
  end
@@ -7,12 +7,12 @@ describe BigBench::Configuration do
7
7
  BigBench.configure = {
8
8
  :duration => 10.seconds,
9
9
  :output => "test.ljson",
10
- :threads => 20
10
+ :users => 20
11
11
  }
12
12
 
13
13
  BigBench.config.duration.should == 10.seconds
14
14
  BigBench.config.output.should == "test.ljson"
15
- BigBench.config.threads.should == 20
15
+ BigBench.config.users.should == 20
16
16
 
17
17
  end
18
18
 
@@ -27,7 +27,7 @@ describe BigBench::Configuration do
27
27
  BigBench.configure = {
28
28
  :duration => 10.seconds,
29
29
  :output => "test.ljson",
30
- :threads => 20
30
+ :users => 20
31
31
  }
32
32
  BigBench.config.should be_valid
33
33
  end