bigbench 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/Gemfile +4 -0
- data/README.textile +81 -0
- data/Rakefile +10 -0
- data/bigbench.gemspec +29 -0
- data/bin/bigbench +6 -0
- data/doc/BigBench.html +631 -0
- data/doc/BigBench/Benchmark.html +328 -0
- data/doc/BigBench/Benchmark/Benchmark.html +431 -0
- data/doc/BigBench/Bot.html +233 -0
- data/doc/BigBench/Configuration.html +312 -0
- data/doc/BigBench/Configuration/Config.html +375 -0
- data/doc/BigBench/Configuration/InvalidOptions.html +217 -0
- data/doc/BigBench/Executor.html +252 -0
- data/doc/BigBench/Executor/InvalidCommand.html +222 -0
- data/doc/BigBench/Fragment.html +404 -0
- data/doc/BigBench/Fragment/Fragment.html +384 -0
- data/doc/BigBench/Output.html +703 -0
- data/doc/BigBench/Runner.html +222 -0
- data/doc/BigBench/Runner/NoBenchmarksDefined.html +217 -0
- data/doc/BigBench/Store.html +639 -0
- data/doc/BigBench/Tracker.html +179 -0
- data/doc/BigBench/Tracker/Tracker.html +273 -0
- data/doc/Float.html +217 -0
- data/doc/Gemfile.html +129 -0
- data/doc/Helpers.html +163 -0
- data/doc/Object.html +181 -0
- data/doc/Rakefile.html +136 -0
- data/doc/TestWebServer.html +247 -0
- data/doc/created.rid +31 -0
- data/doc/images/add.png +0 -0
- data/doc/images/brick.png +0 -0
- data/doc/images/brick_link.png +0 -0
- data/doc/images/bug.png +0 -0
- data/doc/images/bullet_black.png +0 -0
- data/doc/images/bullet_toggle_minus.png +0 -0
- data/doc/images/bullet_toggle_plus.png +0 -0
- data/doc/images/date.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/find.png +0 -0
- data/doc/images/loadingAnimation.gif +0 -0
- data/doc/images/macFFBgHack.png +0 -0
- data/doc/images/package.png +0 -0
- data/doc/images/page_green.png +0 -0
- data/doc/images/page_white_text.png +0 -0
- data/doc/images/page_white_width.png +0 -0
- data/doc/images/plugin.png +0 -0
- data/doc/images/ruby.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/tag_green.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/images/wrench.png +0 -0
- data/doc/images/wrench_orange.png +0 -0
- data/doc/images/zoom.png +0 -0
- data/doc/index.html +122 -0
- data/doc/js/darkfish.js +153 -0
- data/doc/js/jquery.js +18 -0
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/lib/bigbench/help/executor_txt.html +145 -0
- data/doc/rdoc.css +543 -0
- data/doc/table_of_contents.html +263 -0
- data/lib/bigbench.rb +18 -0
- data/lib/bigbench/benchmark.rb +126 -0
- data/lib/bigbench/bot.rb +34 -0
- data/lib/bigbench/configuration.rb +109 -0
- data/lib/bigbench/executor.rb +131 -0
- data/lib/bigbench/float_extensions.rb +6 -0
- data/lib/bigbench/fragment.rb +119 -0
- data/lib/bigbench/help/executor.txt +17 -0
- data/lib/bigbench/initializers.rb +2 -0
- data/lib/bigbench/output.rb +116 -0
- data/lib/bigbench/runner.rb +52 -0
- data/lib/bigbench/store.rb +92 -0
- data/lib/bigbench/tracker.rb +79 -0
- data/lib/bigbench/version.rb +3 -0
- data/spec/benchmark_spec.rb +69 -0
- data/spec/bot_spec.rb +31 -0
- data/spec/configure_spec.rb +39 -0
- data/spec/executor_spec.rb +80 -0
- data/spec/fragment_spec.rb +110 -0
- data/spec/helpers.rb +19 -0
- data/spec/lib/test_web_server.rb +18 -0
- data/spec/runner_spec.rb +40 -0
- data/spec/store_spec.rb +72 -0
- data/spec/tests/local.rb +15 -0
- data/spec/tests/local_invalid.rb +9 -0
- data/spec/tracker_spec.rb +50 -0
- data/spec/webserver_spec.rb +35 -0
- metadata +220 -0
@@ -0,0 +1,131 @@
|
|
1
|
+
module BigBench
|
2
|
+
|
3
|
+
# Dispatches the command line commands to BigBench calls. Available commands are:
|
4
|
+
#
|
5
|
+
# :include: executor.txt
|
6
|
+
module Executor
|
7
|
+
|
8
|
+
COMMANDS = [
|
9
|
+
"--help", "show help",
|
10
|
+
"run local PATH_TO_TEST",
|
11
|
+
"run bots PATH_TO_TEST [REDIS_URL_WITH_PORT REDIS_PASSWORD]",
|
12
|
+
"start bot [REDIS_URL_WITH_PORT REDIS_PASSWORD]",
|
13
|
+
"reset all [REDIS_URL_WITH_PORT REDIS_PASSWORD]"
|
14
|
+
]
|
15
|
+
|
16
|
+
# Is thrown when the command is not known
|
17
|
+
class InvalidCommand < StandardError
|
18
|
+
def message
|
19
|
+
puts "\n\n"
|
20
|
+
puts "Sorry, could not compile your command. Please enter one of the following commands:\n\n"
|
21
|
+
COMMANDS.each{ |command| puts command }
|
22
|
+
puts " "
|
23
|
+
|
24
|
+
"Sorry, could not compile your command."
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# Parses and runs the BigBench funtion that is supplied via the commandline's ARGV. It
|
29
|
+
# throws an <tt>InvalidCommand</tt> exception if the command is unknown.
|
30
|
+
def self.run!(argv)
|
31
|
+
BigBench::Output.start
|
32
|
+
|
33
|
+
check_arguments!(argv)
|
34
|
+
send to_executor_method(argv), argv
|
35
|
+
|
36
|
+
BigBench::Output.done
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
# Maps the incoming ARGV to a method
|
42
|
+
def self.to_executor_method(argv)
|
43
|
+
return "show_help" if argv.first == "--help"
|
44
|
+
"#{argv[0]}_#{argv[1]}".to_sym
|
45
|
+
end
|
46
|
+
|
47
|
+
# Checks if the supplied ARGV has a valid and useable form
|
48
|
+
def self.check_arguments!(argv)
|
49
|
+
raise InvalidCommand.new if argv.nil? or argv.empty?
|
50
|
+
raise InvalidCommand.new unless respond_to?(to_executor_method(argv))
|
51
|
+
true
|
52
|
+
end
|
53
|
+
|
54
|
+
# Runs a local test with the specified test.rb. For this mode no redis is needed.
|
55
|
+
def self.run_local(argv)
|
56
|
+
BigBench.load_test! File.open(argv[2], "rb"){ |file| file.read }
|
57
|
+
BigBench.run!
|
58
|
+
BigBench.write_local_trackings_to_file!
|
59
|
+
end
|
60
|
+
|
61
|
+
# Runs a test on all available bots with the specified test.rb
|
62
|
+
#
|
63
|
+
# bigbench run bots test.rb http://localhost:6379 password
|
64
|
+
#
|
65
|
+
def self.run_bots(argv)
|
66
|
+
|
67
|
+
# Load and validate tests
|
68
|
+
test = File.open(argv[2], "rb"){ |file| file.read }
|
69
|
+
BigBench.load_test!(test)
|
70
|
+
BigBench.config.mode = :bots
|
71
|
+
|
72
|
+
# Initialize redis
|
73
|
+
argv.shift(3)
|
74
|
+
BigBench::Store.setup!(*argv)
|
75
|
+
|
76
|
+
# Load current test to redis & start bots
|
77
|
+
BigBench::Store.test = test
|
78
|
+
BigBench::Store.start
|
79
|
+
BigBench::Output.deployed_test
|
80
|
+
|
81
|
+
# Wait for bots to run the tests
|
82
|
+
@is_running = true
|
83
|
+
timer = Thread.new{
|
84
|
+
sleep(BigBench.duration.to_i)
|
85
|
+
BigBench::Store.stop
|
86
|
+
@is_running = false
|
87
|
+
}
|
88
|
+
|
89
|
+
BigBench::Output.starting_bots_loop
|
90
|
+
loop{
|
91
|
+
bots = BigBench::Store.bots
|
92
|
+
BigBench::Output.running_bots_loop(bots)
|
93
|
+
sleep(1)
|
94
|
+
break if !@is_running and bots.size == 0
|
95
|
+
}
|
96
|
+
|
97
|
+
# Gather trackings from redis and write them to the file
|
98
|
+
BigBench::Output.finished_bots_loop
|
99
|
+
BigBench.write_store_trackings_to_file!
|
100
|
+
BigBench::Store.reset!
|
101
|
+
end
|
102
|
+
|
103
|
+
# Starts the bot that listens for tests on the redis
|
104
|
+
def self.start_bot(argv)
|
105
|
+
argv.shift(2)
|
106
|
+
BigBench::Store.setup!(*argv)
|
107
|
+
|
108
|
+
loop {
|
109
|
+
BigBench::Output.bot_is_checking
|
110
|
+
BigBench::Bot.check_test!
|
111
|
+
sleep(BigBench.config.bot_checks_every.to_i)
|
112
|
+
}
|
113
|
+
end
|
114
|
+
|
115
|
+
# Shows the command line help
|
116
|
+
def self.show_help(argv)
|
117
|
+
File.open("lib/bigbench/help/executor.txt", "rb"){ |file| file.read }
|
118
|
+
end
|
119
|
+
|
120
|
+
# Resets everything
|
121
|
+
def self.reset_all(argv)
|
122
|
+
argv.shift(2)
|
123
|
+
BigBench::Store.setup!(*argv)
|
124
|
+
BigBench::Store.reset!
|
125
|
+
BigBench::Configuration.reset!
|
126
|
+
BigBench::Benchmark.reset!
|
127
|
+
BigBench::Output.reset
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module BigBench
|
2
|
+
|
3
|
+
# A fragment represents a single http request inside a benchmark. It is executed by the benchmark and resides inside the benchmark block in the test
|
4
|
+
# reciepts:
|
5
|
+
#
|
6
|
+
# benchmark "index page" => "http://localhost:3000" do
|
7
|
+
# # Fragments
|
8
|
+
# end
|
9
|
+
#
|
10
|
+
# Possible fragment types are the HTTP verbs, like GET, POST, PUT and DELETE. They look like this:
|
11
|
+
#
|
12
|
+
# get "/"
|
13
|
+
# post "/login/new", { :name => "a name", :password => "secret" }
|
14
|
+
# put "/books", { :name => "A Book title", :rating => 30.4 }
|
15
|
+
# delete "/books/5"
|
16
|
+
#
|
17
|
+
module Fragment
|
18
|
+
|
19
|
+
@fragments = []
|
20
|
+
@benchmark = nil
|
21
|
+
|
22
|
+
class Fragment
|
23
|
+
|
24
|
+
attr_accessor :benchmark
|
25
|
+
attr_accessor :path
|
26
|
+
attr_accessor :method
|
27
|
+
attr_accessor :params
|
28
|
+
attr_accessor :uri
|
29
|
+
|
30
|
+
def initialize benchmark, path, method, params = {}
|
31
|
+
@benchmark, @path, @method, @params = benchmark, path, method, params
|
32
|
+
@uri = URI(@benchmark.uri.to_s + @path)
|
33
|
+
end
|
34
|
+
|
35
|
+
# Initiates the request in the context of a <tt>Net::HTTP.start</tt> block
|
36
|
+
def run!(http)
|
37
|
+
request = case @method
|
38
|
+
when :get then Net::HTTP::Get.new(@uri.request_uri)
|
39
|
+
when :post then Net::HTTP::Post.new(@uri.request_uri)
|
40
|
+
when :put then Net::HTTP::Put.new(@uri.request_uri)
|
41
|
+
when :delete then Net::HTTP::Delete.new(@uri.request_uri)
|
42
|
+
else nil
|
43
|
+
end
|
44
|
+
|
45
|
+
start = Time.now
|
46
|
+
response = http.request(request)
|
47
|
+
stop = Time.now
|
48
|
+
|
49
|
+
track!(start, stop, response)
|
50
|
+
response
|
51
|
+
end
|
52
|
+
|
53
|
+
# Adds the current tracking result as a hash to the benchmark's tracker
|
54
|
+
def track!(start, stop, response)
|
55
|
+
@benchmark.tracker.track(
|
56
|
+
{
|
57
|
+
:elapsed => (stop - benchmark.start).to_f,
|
58
|
+
:start => start.to_f,
|
59
|
+
:stop => stop.to_f,
|
60
|
+
:duration => (stop - start).to_milliseconds,
|
61
|
+
:benchmark => @benchmark.name,
|
62
|
+
:url => @uri.to_s,
|
63
|
+
:path => @uri.request_uri,
|
64
|
+
:method => @method,
|
65
|
+
:status => response.code
|
66
|
+
}
|
67
|
+
)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Performs a GET request to the given url, e.g.
|
72
|
+
#
|
73
|
+
# get "/some/page"
|
74
|
+
#
|
75
|
+
def self.get(path)
|
76
|
+
@fragments << Fragment.new(@benchmark, path, :get, {})
|
77
|
+
end
|
78
|
+
|
79
|
+
# Performs a POST request to the given url, e.g.
|
80
|
+
#
|
81
|
+
# post "/login", { :user => "some@sample.com", :password => "secret" }
|
82
|
+
#
|
83
|
+
def self.post(path, params = {})
|
84
|
+
@fragments << Fragment.new(@benchmark, path, :post, params)
|
85
|
+
end
|
86
|
+
|
87
|
+
# Performs a PUT request to the given url, e.g.
|
88
|
+
#
|
89
|
+
# put "/books", { :book => "Some book content" }
|
90
|
+
#
|
91
|
+
def self.put(path, params = {})
|
92
|
+
@fragments << Fragment.new(@benchmark, path, :put, params)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Performs a DELETE request to the given url, e.g.
|
96
|
+
#
|
97
|
+
# delete "/books/5", { :user => "some@sample.com", :password => "secret" }
|
98
|
+
#
|
99
|
+
def self.delete(path, params = {})
|
100
|
+
@fragments << Fragment.new(@benchmark, path, :delete, params)
|
101
|
+
end
|
102
|
+
|
103
|
+
# Evaluates a benchmark block full of puts and gets and returns an array of fully configured fragments for it
|
104
|
+
def self.parse(benchmark, &block)
|
105
|
+
reset!
|
106
|
+
return [] if block.nil?
|
107
|
+
@benchmark = benchmark
|
108
|
+
module_exec(&block)
|
109
|
+
@fragments
|
110
|
+
end
|
111
|
+
|
112
|
+
# Reset all fragments
|
113
|
+
def self.reset!
|
114
|
+
@benchmark = nil
|
115
|
+
@fragments = []
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
Usage:
|
2
|
+
# Local Testing
|
3
|
+
bigbench run local PATH_TO_TEST # Runs a test from the local machine. No redis needed.
|
4
|
+
|
5
|
+
# Remote Testing
|
6
|
+
bigbench run bots PATH_TO_TEST [REDIS_URL_WITH_PORT REDIS_PASSWORD] # Deploys and starts the test on all bots
|
7
|
+
bigbench start bot [REDIS_URL_WITH_PORT REDIS_PASSWORD] # Starts a bot on a remote machine
|
8
|
+
|
9
|
+
# Reseting
|
10
|
+
bigbench reset all # Resets everything
|
11
|
+
|
12
|
+
# Help
|
13
|
+
bigbench --help || show help # Displays this help
|
14
|
+
|
15
|
+
Options:
|
16
|
+
REDIS_URL_WITH_PORT # Defaults to http://localhost:6379
|
17
|
+
REDIS_PASSWORD # Defaults to no password
|
@@ -0,0 +1,116 @@
|
|
1
|
+
module BigBench
|
2
|
+
|
3
|
+
# This module is used to keep all the command line outputs in a single place. The output module gets notified of the code and
|
4
|
+
# can then do what it want's with this information.
|
5
|
+
module Output
|
6
|
+
|
7
|
+
mattr_accessor :start
|
8
|
+
mattr_accessor :trackings
|
9
|
+
|
10
|
+
@start = Time.now
|
11
|
+
@trackings = 1
|
12
|
+
|
13
|
+
def self.start
|
14
|
+
puts "-> Started BigBench at #{@start = Time.now}\n"
|
15
|
+
$stdout.flush
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.done
|
19
|
+
puts "\n-> Done. Took #{Time.now - @start} seconds."
|
20
|
+
$stdout.flush
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.reset
|
24
|
+
puts "-> Resetting everything."
|
25
|
+
$stdout.flush
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.loaded_tests
|
29
|
+
puts "\n-> Loading\n"
|
30
|
+
puts "Loaded #{BigBench.benchmarks.size} benchmarks. Benchmark will take #{BigBench.duration} seconds"
|
31
|
+
$stdout.flush
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.running_benchmarks
|
35
|
+
puts "\n-> Running\n"
|
36
|
+
puts "Running #{BigBench.benchmarks.size} benchmarks."
|
37
|
+
$stdout.flush
|
38
|
+
|
39
|
+
Thread.new{
|
40
|
+
loop {
|
41
|
+
sleep(1)
|
42
|
+
progress = Time.now - @start
|
43
|
+
remaining = BigBench.duration.to_i - progress
|
44
|
+
percent = ((progress.to_f / BigBench.duration.to_f).to_f * 100).to_i
|
45
|
+
print "\r#{percent}% - #{progress.to_i} seconds elapsed. #{remaining.to_i} seconds remaining."
|
46
|
+
if percent > 99
|
47
|
+
print "\r100% Done \n"
|
48
|
+
break
|
49
|
+
end
|
50
|
+
}
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.finished_running_benchmarks
|
55
|
+
puts "Finished #{BigBench.benchmarks.size} benchmarks."
|
56
|
+
$stdout.flush
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.writing_trackings(count)
|
60
|
+
target = BigBench.config.mode == :bot ? 'redis' : BigBench.config.output
|
61
|
+
puts "\n-> Writing\n"
|
62
|
+
puts "Writing #{@trackings = count} trackings to #{target}."
|
63
|
+
$stdout.flush
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.wrote_trackings(count)
|
67
|
+
percent = ((count.to_f / @trackings.to_f).to_f * 100).to_i
|
68
|
+
$stdout.flush
|
69
|
+
print "\r#{percent}%"
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.finished_writing_trackings(count)
|
73
|
+
print "\r100% Done \n"
|
74
|
+
target = BigBench.config.mode == :bot ? 'redis' : BigBench.config.output
|
75
|
+
puts "\nWrote #{count} trackings to #{target}."
|
76
|
+
$stdout.flush
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.deployed_test
|
80
|
+
puts "Deployed test to the redis store."
|
81
|
+
$stdout.flush
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.starting_bots_loop
|
85
|
+
puts "\n-> Running\n"
|
86
|
+
puts "Running benchmarks on bots."
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.running_bots_loop(bots)
|
90
|
+
progress = Time.now - @start
|
91
|
+
remaining = BigBench.duration.to_i - progress
|
92
|
+
percent = ((progress.to_f / BigBench.duration.to_f).to_f * 100).to_i
|
93
|
+
info = " - Waiting for bots to finish" if percent > 100
|
94
|
+
$stdout.flush
|
95
|
+
print "\r#{percent}% - #{progress.to_i} seconds elapsed. #{remaining.to_i} seconds remaining. #{bots.size} Active Bots#{info} "
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.finished_bots_loop
|
99
|
+
print "\r100% Done \n"
|
100
|
+
puts "Finished bots test."
|
101
|
+
$stdout.flush
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.bot_is_checking
|
105
|
+
puts "Checking for new tests at #{Time.now}"
|
106
|
+
$stdout.flush
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.bot_received_test_instructions
|
110
|
+
puts "\n-> Received\n"
|
111
|
+
puts "Received test instructions\n"
|
112
|
+
$stdout.flush
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module BigBench
|
2
|
+
|
3
|
+
# The runner runs all available benchmarks and returns as soon as all of them are finished. Additionally it allows the loading
|
4
|
+
# of reciepts from files or the redis storage.
|
5
|
+
module Runner
|
6
|
+
|
7
|
+
# Is thrown when no benchmarks are defined
|
8
|
+
class NoBenchmarksDefined < StandardError
|
9
|
+
def message
|
10
|
+
"Sorry, I couldn't find any benchmarks!"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Runs all benchmarks
|
15
|
+
def self.run!
|
16
|
+
raise NoBenchmarksDefined.new if BigBench.benchmarks.empty?
|
17
|
+
|
18
|
+
# Run all benchmarks
|
19
|
+
@running_threads = []
|
20
|
+
BigBench.benchmarks.each do |benchmark|
|
21
|
+
@running_threads << Thread.new{ benchmark.run! }
|
22
|
+
end
|
23
|
+
|
24
|
+
# Wait for them to finish
|
25
|
+
@running_threads.each{ |thread| thread.join }
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
# Runs all initialized benchmarks
|
31
|
+
def self.run!
|
32
|
+
BigBench::Output.running_benchmarks
|
33
|
+
Runner.run!
|
34
|
+
BigBench::Output.finished_running_benchmarks
|
35
|
+
end
|
36
|
+
|
37
|
+
# Loads a test from a string file that is either parsed from a local file or retreived from the key-value store.
|
38
|
+
#
|
39
|
+
# benchmark_string = 'benchmark "index page" => "http://localhost:3000" do
|
40
|
+
# get "/"
|
41
|
+
# end'
|
42
|
+
#
|
43
|
+
# BigBench.load_test!(benchmark_string)
|
44
|
+
#
|
45
|
+
def self.load_test!(test)
|
46
|
+
BigBench::Benchmark.reset!
|
47
|
+
eval(test)
|
48
|
+
check_config!
|
49
|
+
BigBench::Output.loaded_tests
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|