bigbench 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. data/.gitignore +5 -0
  2. data/Gemfile +4 -0
  3. data/README.textile +81 -0
  4. data/Rakefile +10 -0
  5. data/bigbench.gemspec +29 -0
  6. data/bin/bigbench +6 -0
  7. data/doc/BigBench.html +631 -0
  8. data/doc/BigBench/Benchmark.html +328 -0
  9. data/doc/BigBench/Benchmark/Benchmark.html +431 -0
  10. data/doc/BigBench/Bot.html +233 -0
  11. data/doc/BigBench/Configuration.html +312 -0
  12. data/doc/BigBench/Configuration/Config.html +375 -0
  13. data/doc/BigBench/Configuration/InvalidOptions.html +217 -0
  14. data/doc/BigBench/Executor.html +252 -0
  15. data/doc/BigBench/Executor/InvalidCommand.html +222 -0
  16. data/doc/BigBench/Fragment.html +404 -0
  17. data/doc/BigBench/Fragment/Fragment.html +384 -0
  18. data/doc/BigBench/Output.html +703 -0
  19. data/doc/BigBench/Runner.html +222 -0
  20. data/doc/BigBench/Runner/NoBenchmarksDefined.html +217 -0
  21. data/doc/BigBench/Store.html +639 -0
  22. data/doc/BigBench/Tracker.html +179 -0
  23. data/doc/BigBench/Tracker/Tracker.html +273 -0
  24. data/doc/Float.html +217 -0
  25. data/doc/Gemfile.html +129 -0
  26. data/doc/Helpers.html +163 -0
  27. data/doc/Object.html +181 -0
  28. data/doc/Rakefile.html +136 -0
  29. data/doc/TestWebServer.html +247 -0
  30. data/doc/created.rid +31 -0
  31. data/doc/images/add.png +0 -0
  32. data/doc/images/brick.png +0 -0
  33. data/doc/images/brick_link.png +0 -0
  34. data/doc/images/bug.png +0 -0
  35. data/doc/images/bullet_black.png +0 -0
  36. data/doc/images/bullet_toggle_minus.png +0 -0
  37. data/doc/images/bullet_toggle_plus.png +0 -0
  38. data/doc/images/date.png +0 -0
  39. data/doc/images/delete.png +0 -0
  40. data/doc/images/find.png +0 -0
  41. data/doc/images/loadingAnimation.gif +0 -0
  42. data/doc/images/macFFBgHack.png +0 -0
  43. data/doc/images/package.png +0 -0
  44. data/doc/images/page_green.png +0 -0
  45. data/doc/images/page_white_text.png +0 -0
  46. data/doc/images/page_white_width.png +0 -0
  47. data/doc/images/plugin.png +0 -0
  48. data/doc/images/ruby.png +0 -0
  49. data/doc/images/tag_blue.png +0 -0
  50. data/doc/images/tag_green.png +0 -0
  51. data/doc/images/transparent.png +0 -0
  52. data/doc/images/wrench.png +0 -0
  53. data/doc/images/wrench_orange.png +0 -0
  54. data/doc/images/zoom.png +0 -0
  55. data/doc/index.html +122 -0
  56. data/doc/js/darkfish.js +153 -0
  57. data/doc/js/jquery.js +18 -0
  58. data/doc/js/navigation.js +142 -0
  59. data/doc/js/search.js +94 -0
  60. data/doc/js/search_index.js +1 -0
  61. data/doc/js/searcher.js +228 -0
  62. data/doc/lib/bigbench/help/executor_txt.html +145 -0
  63. data/doc/rdoc.css +543 -0
  64. data/doc/table_of_contents.html +263 -0
  65. data/lib/bigbench.rb +18 -0
  66. data/lib/bigbench/benchmark.rb +126 -0
  67. data/lib/bigbench/bot.rb +34 -0
  68. data/lib/bigbench/configuration.rb +109 -0
  69. data/lib/bigbench/executor.rb +131 -0
  70. data/lib/bigbench/float_extensions.rb +6 -0
  71. data/lib/bigbench/fragment.rb +119 -0
  72. data/lib/bigbench/help/executor.txt +17 -0
  73. data/lib/bigbench/initializers.rb +2 -0
  74. data/lib/bigbench/output.rb +116 -0
  75. data/lib/bigbench/runner.rb +52 -0
  76. data/lib/bigbench/store.rb +92 -0
  77. data/lib/bigbench/tracker.rb +79 -0
  78. data/lib/bigbench/version.rb +3 -0
  79. data/spec/benchmark_spec.rb +69 -0
  80. data/spec/bot_spec.rb +31 -0
  81. data/spec/configure_spec.rb +39 -0
  82. data/spec/executor_spec.rb +80 -0
  83. data/spec/fragment_spec.rb +110 -0
  84. data/spec/helpers.rb +19 -0
  85. data/spec/lib/test_web_server.rb +18 -0
  86. data/spec/runner_spec.rb +40 -0
  87. data/spec/store_spec.rb +72 -0
  88. data/spec/tests/local.rb +15 -0
  89. data/spec/tests/local_invalid.rb +9 -0
  90. data/spec/tracker_spec.rb +50 -0
  91. data/spec/webserver_spec.rb +35 -0
  92. metadata +220 -0
@@ -0,0 +1,131 @@
1
+ module BigBench
2
+
3
+ # Dispatches the command line commands to BigBench calls. Available commands are:
4
+ #
5
+ # :include: executor.txt
6
+ module Executor
7
+
8
+ COMMANDS = [
9
+ "--help", "show help",
10
+ "run local PATH_TO_TEST",
11
+ "run bots PATH_TO_TEST [REDIS_URL_WITH_PORT REDIS_PASSWORD]",
12
+ "start bot [REDIS_URL_WITH_PORT REDIS_PASSWORD]",
13
+ "reset all [REDIS_URL_WITH_PORT REDIS_PASSWORD]"
14
+ ]
15
+
16
+ # Is thrown when the command is not known
17
+ class InvalidCommand < StandardError
18
+ def message
19
+ puts "\n\n"
20
+ puts "Sorry, could not compile your command. Please enter one of the following commands:\n\n"
21
+ COMMANDS.each{ |command| puts command }
22
+ puts " "
23
+
24
+ "Sorry, could not compile your command."
25
+ end
26
+ end
27
+
28
+ # Parses and runs the BigBench funtion that is supplied via the commandline's ARGV. It
29
+ # throws an <tt>InvalidCommand</tt> exception if the command is unknown.
30
+ def self.run!(argv)
31
+ BigBench::Output.start
32
+
33
+ check_arguments!(argv)
34
+ send to_executor_method(argv), argv
35
+
36
+ BigBench::Output.done
37
+ end
38
+
39
+ private
40
+
41
+ # Maps the incoming ARGV to a method
42
+ def self.to_executor_method(argv)
43
+ return "show_help" if argv.first == "--help"
44
+ "#{argv[0]}_#{argv[1]}".to_sym
45
+ end
46
+
47
+ # Checks if the supplied ARGV has a valid and useable form
48
+ def self.check_arguments!(argv)
49
+ raise InvalidCommand.new if argv.nil? or argv.empty?
50
+ raise InvalidCommand.new unless respond_to?(to_executor_method(argv))
51
+ true
52
+ end
53
+
54
+ # Runs a local test with the specified test.rb. For this mode no redis is needed.
55
+ def self.run_local(argv)
56
+ BigBench.load_test! File.open(argv[2], "rb"){ |file| file.read }
57
+ BigBench.run!
58
+ BigBench.write_local_trackings_to_file!
59
+ end
60
+
61
+ # Runs a test on all available bots with the specified test.rb
62
+ #
63
+ # bigbench run bots test.rb http://localhost:6379 password
64
+ #
65
+ def self.run_bots(argv)
66
+
67
+ # Load and validate tests
68
+ test = File.open(argv[2], "rb"){ |file| file.read }
69
+ BigBench.load_test!(test)
70
+ BigBench.config.mode = :bots
71
+
72
+ # Initialize redis
73
+ argv.shift(3)
74
+ BigBench::Store.setup!(*argv)
75
+
76
+ # Load current test to redis & start bots
77
+ BigBench::Store.test = test
78
+ BigBench::Store.start
79
+ BigBench::Output.deployed_test
80
+
81
+ # Wait for bots to run the tests
82
+ @is_running = true
83
+ timer = Thread.new{
84
+ sleep(BigBench.duration.to_i)
85
+ BigBench::Store.stop
86
+ @is_running = false
87
+ }
88
+
89
+ BigBench::Output.starting_bots_loop
90
+ loop{
91
+ bots = BigBench::Store.bots
92
+ BigBench::Output.running_bots_loop(bots)
93
+ sleep(1)
94
+ break if !@is_running and bots.size == 0
95
+ }
96
+
97
+ # Gather trackings from redis and write them to the file
98
+ BigBench::Output.finished_bots_loop
99
+ BigBench.write_store_trackings_to_file!
100
+ BigBench::Store.reset!
101
+ end
102
+
103
+ # Starts the bot that listens for tests on the redis
104
+ def self.start_bot(argv)
105
+ argv.shift(2)
106
+ BigBench::Store.setup!(*argv)
107
+
108
+ loop {
109
+ BigBench::Output.bot_is_checking
110
+ BigBench::Bot.check_test!
111
+ sleep(BigBench.config.bot_checks_every.to_i)
112
+ }
113
+ end
114
+
115
+ # Shows the command line help
116
+ def self.show_help(argv)
117
+ File.open("lib/bigbench/help/executor.txt", "rb"){ |file| file.read }
118
+ end
119
+
120
+ # Resets everything
121
+ def self.reset_all(argv)
122
+ argv.shift(2)
123
+ BigBench::Store.setup!(*argv)
124
+ BigBench::Store.reset!
125
+ BigBench::Configuration.reset!
126
+ BigBench::Benchmark.reset!
127
+ BigBench::Output.reset
128
+ end
129
+
130
+ end
131
+ end
@@ -0,0 +1,6 @@
1
+ # Adds the <tt>to_milliseconds</tt> method to any float value
2
+ class Float
3
+ def to_milliseconds
4
+ (1000 * self).to_i
5
+ end
6
+ end
@@ -0,0 +1,119 @@
1
+ module BigBench
2
+
3
+ # A fragment represents a single http request inside a benchmark. It is executed by the benchmark and resides inside the benchmark block in the test
4
+ # reciepts:
5
+ #
6
+ # benchmark "index page" => "http://localhost:3000" do
7
+ # # Fragments
8
+ # end
9
+ #
10
+ # Possible fragment types are the HTTP verbs, like GET, POST, PUT and DELETE. They look like this:
11
+ #
12
+ # get "/"
13
+ # post "/login/new", { :name => "a name", :password => "secret" }
14
+ # put "/books", { :name => "A Book title", :rating => 30.4 }
15
+ # delete "/books/5"
16
+ #
17
+ module Fragment
18
+
19
+ @fragments = []
20
+ @benchmark = nil
21
+
22
+ class Fragment
23
+
24
+ attr_accessor :benchmark
25
+ attr_accessor :path
26
+ attr_accessor :method
27
+ attr_accessor :params
28
+ attr_accessor :uri
29
+
30
+ def initialize benchmark, path, method, params = {}
31
+ @benchmark, @path, @method, @params = benchmark, path, method, params
32
+ @uri = URI(@benchmark.uri.to_s + @path)
33
+ end
34
+
35
+ # Initiates the request in the context of a <tt>Net::HTTP.start</tt> block
36
+ def run!(http)
37
+ request = case @method
38
+ when :get then Net::HTTP::Get.new(@uri.request_uri)
39
+ when :post then Net::HTTP::Post.new(@uri.request_uri)
40
+ when :put then Net::HTTP::Put.new(@uri.request_uri)
41
+ when :delete then Net::HTTP::Delete.new(@uri.request_uri)
42
+ else nil
43
+ end
44
+
45
+ start = Time.now
46
+ response = http.request(request)
47
+ stop = Time.now
48
+
49
+ track!(start, stop, response)
50
+ response
51
+ end
52
+
53
+ # Adds the current tracking result as a hash to the benchmark's tracker
54
+ def track!(start, stop, response)
55
+ @benchmark.tracker.track(
56
+ {
57
+ :elapsed => (stop - benchmark.start).to_f,
58
+ :start => start.to_f,
59
+ :stop => stop.to_f,
60
+ :duration => (stop - start).to_milliseconds,
61
+ :benchmark => @benchmark.name,
62
+ :url => @uri.to_s,
63
+ :path => @uri.request_uri,
64
+ :method => @method,
65
+ :status => response.code
66
+ }
67
+ )
68
+ end
69
+ end
70
+
71
+ # Performs a GET request to the given url, e.g.
72
+ #
73
+ # get "/some/page"
74
+ #
75
+ def self.get(path)
76
+ @fragments << Fragment.new(@benchmark, path, :get, {})
77
+ end
78
+
79
+ # Performs a POST request to the given url, e.g.
80
+ #
81
+ # post "/login", { :user => "some@sample.com", :password => "secret" }
82
+ #
83
+ def self.post(path, params = {})
84
+ @fragments << Fragment.new(@benchmark, path, :post, params)
85
+ end
86
+
87
+ # Performs a PUT request to the given url, e.g.
88
+ #
89
+ # put "/books", { :book => "Some book content" }
90
+ #
91
+ def self.put(path, params = {})
92
+ @fragments << Fragment.new(@benchmark, path, :put, params)
93
+ end
94
+
95
+ # Performs a DELETE request to the given url, e.g.
96
+ #
97
+ # delete "/books/5", { :user => "some@sample.com", :password => "secret" }
98
+ #
99
+ def self.delete(path, params = {})
100
+ @fragments << Fragment.new(@benchmark, path, :delete, params)
101
+ end
102
+
103
+ # Evaluates a benchmark block full of puts and gets and returns an array of fully configured fragments for it
104
+ def self.parse(benchmark, &block)
105
+ reset!
106
+ return [] if block.nil?
107
+ @benchmark = benchmark
108
+ module_exec(&block)
109
+ @fragments
110
+ end
111
+
112
+ # Reset all fragments
113
+ def self.reset!
114
+ @benchmark = nil
115
+ @fragments = []
116
+ end
117
+
118
+ end
119
+ end
@@ -0,0 +1,17 @@
1
+ Usage:
2
+ # Local Testing
3
+ bigbench run local PATH_TO_TEST # Runs a test from the local machine. No redis needed.
4
+
5
+ # Remote Testing
6
+ bigbench run bots PATH_TO_TEST [REDIS_URL_WITH_PORT REDIS_PASSWORD] # Deploys and starts the test on all bots
7
+ bigbench start bot [REDIS_URL_WITH_PORT REDIS_PASSWORD] # Starts a bot on a remote machine
8
+
9
+ # Reseting
10
+ bigbench reset all # Resets everything
11
+
12
+ # Help
13
+ bigbench --help || show help # Displays this help
14
+
15
+ Options:
16
+ REDIS_URL_WITH_PORT # Defaults to http://localhost:6379
17
+ REDIS_PASSWORD # Defaults to no password
@@ -0,0 +1,2 @@
1
+ # ActiveSupport threading bug fix
2
+ { :unthreaded => :hash }.to_json
@@ -0,0 +1,116 @@
1
+ module BigBench
2
+
3
+ # This module is used to keep all the command line outputs in a single place. The output module gets notified of the code and
4
+ # can then do what it want's with this information.
5
+ module Output
6
+
7
+ mattr_accessor :start
8
+ mattr_accessor :trackings
9
+
10
+ @start = Time.now
11
+ @trackings = 1
12
+
13
+ def self.start
14
+ puts "-> Started BigBench at #{@start = Time.now}\n"
15
+ $stdout.flush
16
+ end
17
+
18
+ def self.done
19
+ puts "\n-> Done. Took #{Time.now - @start} seconds."
20
+ $stdout.flush
21
+ end
22
+
23
+ def self.reset
24
+ puts "-> Resetting everything."
25
+ $stdout.flush
26
+ end
27
+
28
+ def self.loaded_tests
29
+ puts "\n-> Loading\n"
30
+ puts "Loaded #{BigBench.benchmarks.size} benchmarks. Benchmark will take #{BigBench.duration} seconds"
31
+ $stdout.flush
32
+ end
33
+
34
+ def self.running_benchmarks
35
+ puts "\n-> Running\n"
36
+ puts "Running #{BigBench.benchmarks.size} benchmarks."
37
+ $stdout.flush
38
+
39
+ Thread.new{
40
+ loop {
41
+ sleep(1)
42
+ progress = Time.now - @start
43
+ remaining = BigBench.duration.to_i - progress
44
+ percent = ((progress.to_f / BigBench.duration.to_f).to_f * 100).to_i
45
+ print "\r#{percent}% - #{progress.to_i} seconds elapsed. #{remaining.to_i} seconds remaining."
46
+ if percent > 99
47
+ print "\r100% Done \n"
48
+ break
49
+ end
50
+ }
51
+ }
52
+ end
53
+
54
+ def self.finished_running_benchmarks
55
+ puts "Finished #{BigBench.benchmarks.size} benchmarks."
56
+ $stdout.flush
57
+ end
58
+
59
+ def self.writing_trackings(count)
60
+ target = BigBench.config.mode == :bot ? 'redis' : BigBench.config.output
61
+ puts "\n-> Writing\n"
62
+ puts "Writing #{@trackings = count} trackings to #{target}."
63
+ $stdout.flush
64
+ end
65
+
66
+ def self.wrote_trackings(count)
67
+ percent = ((count.to_f / @trackings.to_f).to_f * 100).to_i
68
+ $stdout.flush
69
+ print "\r#{percent}%"
70
+ end
71
+
72
+ def self.finished_writing_trackings(count)
73
+ print "\r100% Done \n"
74
+ target = BigBench.config.mode == :bot ? 'redis' : BigBench.config.output
75
+ puts "\nWrote #{count} trackings to #{target}."
76
+ $stdout.flush
77
+ end
78
+
79
+ def self.deployed_test
80
+ puts "Deployed test to the redis store."
81
+ $stdout.flush
82
+ end
83
+
84
+ def self.starting_bots_loop
85
+ puts "\n-> Running\n"
86
+ puts "Running benchmarks on bots."
87
+ end
88
+
89
+ def self.running_bots_loop(bots)
90
+ progress = Time.now - @start
91
+ remaining = BigBench.duration.to_i - progress
92
+ percent = ((progress.to_f / BigBench.duration.to_f).to_f * 100).to_i
93
+ info = " - Waiting for bots to finish" if percent > 100
94
+ $stdout.flush
95
+ print "\r#{percent}% - #{progress.to_i} seconds elapsed. #{remaining.to_i} seconds remaining. #{bots.size} Active Bots#{info} "
96
+ end
97
+
98
+ def self.finished_bots_loop
99
+ print "\r100% Done \n"
100
+ puts "Finished bots test."
101
+ $stdout.flush
102
+ end
103
+
104
+ def self.bot_is_checking
105
+ puts "Checking for new tests at #{Time.now}"
106
+ $stdout.flush
107
+ end
108
+
109
+ def self.bot_received_test_instructions
110
+ puts "\n-> Received\n"
111
+ puts "Received test instructions\n"
112
+ $stdout.flush
113
+ end
114
+
115
+ end
116
+ end
@@ -0,0 +1,52 @@
1
+ module BigBench
2
+
3
+ # The runner runs all available benchmarks and returns as soon as all of them are finished. Additionally it allows the loading
4
+ # of reciepts from files or the redis storage.
5
+ module Runner
6
+
7
+ # Is thrown when no benchmarks are defined
8
+ class NoBenchmarksDefined < StandardError
9
+ def message
10
+ "Sorry, I couldn't find any benchmarks!"
11
+ end
12
+ end
13
+
14
+ # Runs all benchmarks
15
+ def self.run!
16
+ raise NoBenchmarksDefined.new if BigBench.benchmarks.empty?
17
+
18
+ # Run all benchmarks
19
+ @running_threads = []
20
+ BigBench.benchmarks.each do |benchmark|
21
+ @running_threads << Thread.new{ benchmark.run! }
22
+ end
23
+
24
+ # Wait for them to finish
25
+ @running_threads.each{ |thread| thread.join }
26
+ end
27
+
28
+ end
29
+
30
+ # Runs all initialized benchmarks
31
+ def self.run!
32
+ BigBench::Output.running_benchmarks
33
+ Runner.run!
34
+ BigBench::Output.finished_running_benchmarks
35
+ end
36
+
37
+ # Loads a test from a string file that is either parsed from a local file or retreived from the key-value store.
38
+ #
39
+ # benchmark_string = 'benchmark "index page" => "http://localhost:3000" do
40
+ # get "/"
41
+ # end'
42
+ #
43
+ # BigBench.load_test!(benchmark_string)
44
+ #
45
+ def self.load_test!(test)
46
+ BigBench::Benchmark.reset!
47
+ eval(test)
48
+ check_config!
49
+ BigBench::Output.loaded_tests
50
+ end
51
+
52
+ end