solutious-stella 0.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/CHANGES.txt +36 -0
  2. data/README.textile +162 -0
  3. data/Rakefile +88 -0
  4. data/bin/stella +12 -0
  5. data/bin/stella.bat +12 -0
  6. data/lib/daemonize.rb +56 -0
  7. data/lib/pcaplet.rb +180 -0
  8. data/lib/stella.rb +101 -0
  9. data/lib/stella/adapter/ab.rb +337 -0
  10. data/lib/stella/adapter/base.rb +106 -0
  11. data/lib/stella/adapter/httperf.rb +305 -0
  12. data/lib/stella/adapter/pcap_watcher.rb +221 -0
  13. data/lib/stella/adapter/proxy_watcher.rb +76 -0
  14. data/lib/stella/adapter/siege.rb +341 -0
  15. data/lib/stella/cli.rb +258 -0
  16. data/lib/stella/cli/agents.rb +73 -0
  17. data/lib/stella/cli/base.rb +55 -0
  18. data/lib/stella/cli/language.rb +18 -0
  19. data/lib/stella/cli/localtest.rb +78 -0
  20. data/lib/stella/cli/sysinfo.rb +16 -0
  21. data/lib/stella/cli/watch.rb +278 -0
  22. data/lib/stella/command/base.rb +40 -0
  23. data/lib/stella/command/localtest.rb +358 -0
  24. data/lib/stella/data/domain.rb +82 -0
  25. data/lib/stella/data/http.rb +131 -0
  26. data/lib/stella/logger.rb +84 -0
  27. data/lib/stella/response.rb +85 -0
  28. data/lib/stella/storable.rb +201 -0
  29. data/lib/stella/support.rb +276 -0
  30. data/lib/stella/sysinfo.rb +257 -0
  31. data/lib/stella/test/definition.rb +79 -0
  32. data/lib/stella/test/run/summary.rb +70 -0
  33. data/lib/stella/test/stats.rb +114 -0
  34. data/lib/stella/text.rb +64 -0
  35. data/lib/stella/text/resource.rb +38 -0
  36. data/lib/utils/crypto-key.rb +84 -0
  37. data/lib/utils/domainutil.rb +47 -0
  38. data/lib/utils/escape.rb +302 -0
  39. data/lib/utils/fileutil.rb +78 -0
  40. data/lib/utils/httputil.rb +266 -0
  41. data/lib/utils/mathutil.rb +15 -0
  42. data/lib/utils/stats.rb +88 -0
  43. data/lib/utils/textgraph.rb +267 -0
  44. data/lib/utils/timerutil.rb +58 -0
  45. data/lib/win32/Console.rb +970 -0
  46. data/lib/win32/Console/ANSI.rb +305 -0
  47. data/support/kvm.h +91 -0
  48. data/support/ruby-pcap-takuma-notes.txt +19 -0
  49. data/support/ruby-pcap-takuma-patch.txt +30 -0
  50. data/support/text/en.yaml +80 -0
  51. data/support/text/nl.yaml +7 -0
  52. data/support/useragents.txt +75 -0
  53. data/tests/01-util_test.rb +0 -0
  54. data/tests/02-stella-util_test.rb +42 -0
  55. data/tests/10-stella_test.rb +104 -0
  56. data/tests/11-stella-storable_test.rb +68 -0
  57. data/tests/60-stella-command_test.rb +248 -0
  58. data/tests/80-stella-cli_test.rb +45 -0
  59. data/tests/spec-helper.rb +31 -0
  60. metadata +165 -0
@@ -0,0 +1,73 @@
1
+
2
+
3
+
4
+ module Stella
5
+ class CLI
6
+
7
+ class Agents < Stella::CLI::Base
8
+
9
+ attr_accessor :full
10
+ attr_accessor :list
11
+ attr_accessor :search
12
+ attr_accessor :help
13
+
14
+ def initialize(adapter)
15
+ super(adapter)
16
+ @full = false
17
+ @list = false
18
+ @help = false
19
+ end
20
+
21
+ def run
22
+ process_options
23
+
24
+ if @help
25
+ process_options(:display)
26
+ return
27
+ end
28
+
29
+ # The LocalTest command is the keeper of the user agents
30
+ localtest = Stella::LocalTest.new
31
+
32
+ agents = []
33
+ all_agents = localtest.available_agents
34
+ all_agents.each_pair do |key,value|
35
+ if (@full)
36
+ value.each do |full_value|
37
+ agent = full_value.join(' ')
38
+ agents << agent if (!@search || agent.to_s.match(/#{search}/i))
39
+ end
40
+ else
41
+ agents << key.to_s if (!@search || key.to_s.match(/#{search}/i))
42
+ end
43
+ end
44
+
45
+ msg = (@list) ? agents.uniq.sort.join("\n") : Stella::TEXT.msg(:agents_count_message, agents.uniq.size)
46
+ puts msg
47
+ end
48
+
49
+ def process_options(display=false)
50
+
51
+ opts = OptionParser.new
52
+ opts.banner = Stella::TEXT.msg(:option_help_usage)
53
+ opts.on('-h', '--help', Stella::TEXT.msg(:option_help_help)) { @help = true }
54
+ opts.on('-f', '--full', Stella::TEXT.msg(:agents_option_full)) { @full = true }
55
+ opts.on('-l', '--list', Stella::TEXT.msg(:agents_option_list)) { @list = true }
56
+ # TODO: display agents based on shortnames. This is important to maintain continuity with the stella option.
57
+ #opts.on('-a', '--agent', Stella::TEXT.msg(:agents_option_list)) { @list = true }
58
+ opts.on('-s', '--search=S', String, Stella::TEXT.msg(:agents_option_search)) { |v| @search = v }
59
+
60
+ opts.parse!(@arguments)
61
+
62
+ if display
63
+ Stella::LOGGER.info opts
64
+ return
65
+ end
66
+
67
+ end
68
+
69
+ end
70
+
71
+ @@commands['agents'] = Stella::CLI::Agents
72
+ end
73
+ end
@@ -0,0 +1,55 @@
1
+ module Stella
2
+ class CLI
3
+ # Stella::CLI::Base
4
+ #
5
+ # A base case for the command line interface classes. All Stella::CLI
6
+ # classes should be based on this class. Otherwise great destruction could occur.
7
+ class Base
8
+ attr_reader :adapter
9
+ attr_accessor :stella_options
10
+ attr_accessor :arguments
11
+ attr_accessor :working_directory
12
+
13
+ def initialize(adapter)
14
+ @adapter_name = adapter
15
+ @options = OpenStruct.new
16
+
17
+ # There is a bug in Ruby 1.8.6 where a trapped SIGINT will hang.
18
+ # This workaround is from: http://redmine.ruby-lang.org/issues/show/362
19
+ # It works in Ruby 1.9 and JRuby as well.
20
+ # NEW WARNING: This puts the process into a new thread which somehow
21
+ # prevents Pcap from reporting on UDP/DNS packets (TCP/HTTP is unaffected).
22
+ # I left this here as an example of how not to it. Incidentally,
23
+ # "rescue Interrupt" seems to be working fine now.
24
+ # See also: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/220937
25
+ #@killer = Thread.new do
26
+ # puts "#{$/}Exiting...#{$/}"
27
+ # Thread.stop
28
+ # Thread.main.join(1)
29
+ # exit 0
30
+ #end
31
+ #
32
+ # Note that this is just a default. Any class that implements another
33
+ # Signal.trap will override this.
34
+ #Signal.trap('INT') do
35
+ # @killer.call
36
+ # exit 0
37
+ #end
38
+ end
39
+ end
40
+ end
41
+ end
42
+
43
+ __END__
44
+
45
+ TODO: Investigate frylock style definition
46
+ include Stella::CLI::Base
47
+
48
+ before do
49
+ # stuff that would go in initialize
50
+ end
51
+
52
+ command 'sysinfo' do
53
+ puts Stella::SYSINFO.to_yaml(:headers)
54
+ end
55
+
@@ -0,0 +1,18 @@
1
+
2
+
3
+
4
+ module Stella
5
+ class CLI
6
+ class Language < Stella::CLI::Base
7
+
8
+
9
+ def run
10
+ languages = Stella::TEXT.available_languages
11
+ puts Stella::TEXT.msg(:text_available_languages, languages.map { |l| "#{l[:language]} " })
12
+ end
13
+
14
+ end
15
+
16
+ @@commands['lang'] = Stella::CLI::Language
17
+ end
18
+ end
@@ -0,0 +1,78 @@
1
+
2
+
3
+ module Stella
4
+ class CLI
5
+ # Stella::CLI::LocalTest
6
+ #
7
+ # A wrapper that takes the command line input and makes it appropriate for
8
+ # calling an instance of Stella::LocalTest. Then it calls that instance!
9
+ class LocalTest < Stella::CLI::Base
10
+
11
+ attr_reader :testdef
12
+
13
+ def initialize(adapter)
14
+ super(adapter)
15
+ @testdef = Stella::Test::Definition.new
16
+
17
+ if (adapter == 'ab')
18
+ @adapter = Stella::Adapter::ApacheBench.new
19
+ elsif (adapter == 'siege')
20
+ @adapter = Stella::Adapter::Siege.new
21
+ elsif (adapter == 'httperf')
22
+ @adapter = Stella::Adapter::Httperf.new
23
+ else
24
+ raise UnknownValue.new(adapter)
25
+ end
26
+
27
+ @driver = Stella::LocalTest.new
28
+ end
29
+
30
+
31
+ def run
32
+ process_stella_options
33
+
34
+ @adapter.process_arguments(@arguments)
35
+
36
+ @adapter.arguments = @arguments
37
+
38
+ @testdef.vusers = @adapter.vusers
39
+ @testdef.requests = @adapter.requests
40
+
41
+ @driver.adapter = @adapter
42
+ @driver.testdef = @testdef
43
+
44
+ @driver.working_directory = @working_directory
45
+
46
+ @driver.run
47
+ end
48
+
49
+
50
+
51
+ # process_stella_options
52
+ #
53
+ # Populates @testdef with values from @stella_options
54
+ def process_stella_options
55
+ @testdef.repetitions = @stella_options.repetitions
56
+ @testdef.sleep = @stella_options.sleep
57
+ @testdef.warmup = @stella_options.warmup
58
+ @testdef.rampup = @stella_options.rampup
59
+ @testdef.agents = @stella_options.agents
60
+ @testdef.message = @stella_options.message
61
+
62
+
63
+ @driver.quiet = @stella_options.quiet
64
+ @driver.verbose = @stella_options.verbose
65
+ @driver.format = @stella_options.format || 'yaml'
66
+
67
+ end
68
+
69
+
70
+
71
+ end
72
+
73
+
74
+ @@commands['ab'] = Stella::CLI::LocalTest
75
+ @@commands['siege'] = Stella::CLI::LocalTest
76
+ @@commands['httperf'] = Stella::CLI::LocalTest
77
+ end
78
+ end
@@ -0,0 +1,16 @@
1
+
2
+ module Stella
3
+ class CLI
4
+ class SystemInfo < Stella::CLI::Base
5
+
6
+
7
+ def run
8
+ puts Stella::SYSINFO
9
+ end
10
+
11
+ end
12
+
13
+ @@commands['sysinfo'] = Stella::CLI::SystemInfo
14
+ end
15
+ end
16
+
@@ -0,0 +1,278 @@
1
+
2
+ # TODO: Record cookies.
3
+ # TODO: Investigate packetfu: http://code.google.com/p/packetfu/
4
+ # TODO: Investigate Winpcap (http://www.winpcap.org/) and libpcap on Windows
5
+
6
+ module Stella
7
+ class CLI
8
+ class Watch < Stella::CLI::Base
9
+
10
+
11
+ def run
12
+ @options = process_arguments(@arguments)
13
+
14
+ if can_pcap?(@options[:usepcap])
15
+ require 'stella/adapter/pcap_watcher'
16
+ @watcher = Stella::Adapter::PcapWatcher.new(@options)
17
+
18
+ else
19
+ require 'stella/adapter/proxy_watcher'
20
+ @watcher = Stella::Adapter::ProxyWatcher.new(@options)
21
+
22
+ # if can_pcap? returned false, but pcap was requested then we'll
23
+ # call check_pcap to raise the reason why it didn't load.
24
+ if @options[:usepcap]
25
+ check_pcap
26
+ exit 0
27
+ end
28
+ end
29
+
30
+ # We use an observer model so the watcher class will notify us
31
+ # when they have new data. They call the update method below.
32
+ @watcher.add_observer(self)
33
+
34
+ if @options[:record]
35
+
36
+ @record_filepath = generate_record_filepath
37
+ Stella::LOGGER.info("Writing to #{@record_filepath}")
38
+
39
+ if File.exists?(@record_filepath)
40
+ Stella::LOGGER.error("#{@record_filepath} exists")
41
+ if @stella_options.force
42
+ Stella::LOGGER.error("But I'll overwrite it and continue because you forced me too!")
43
+ else
44
+ exit 1
45
+ end
46
+ end
47
+
48
+ end
49
+
50
+ Stella::LOGGER.info("Filter: #{@options[:filter]}") if @options[:filter]
51
+ Stella::LOGGER.info("Domain: #{@options[:domain]}") if @options[:domain]
52
+
53
+ # Turn wildcards into regular expressions
54
+ @options[:filter].gsub!('*', '.*') if @options[:filter]
55
+ @options[:domain].gsub!('*', '.*') if @options[:domain]
56
+
57
+ # Used to calculated user think times for session output
58
+ @think_time = 0
59
+
60
+ @watcher.run
61
+
62
+ end
63
+
64
+
65
+
66
+ # update
67
+ #
68
+ # This method is called from the watcher class when data is updated.
69
+ # +service+ is one of: domain, http_request, http_response
70
+ # +data+ is a string of TCP packet data. The format depends on the value of +service+.
71
+ def update(service, data_object)
72
+
73
+ begin
74
+ if @options[:record] && !@file_created_already
75
+
76
+ if File.exists?(@record_filepath)
77
+ if @stella_options.force
78
+ @record_file = FileUtil.create_file(@record_filepath, 'w', '.', :force)
79
+ else
80
+ exit 1
81
+ end
82
+ else
83
+ @record_file = FileUtil.create_file(@record_filepath, 'w', '.')
84
+ end
85
+
86
+ raise StellaError.new("Cannot open: #{@record_filepath}") unless @record_file
87
+
88
+ @file_created_already = true
89
+ end
90
+ rescue => ex
91
+ raise StellaError.new("Error creating file: #{ex.message}")
92
+ end
93
+
94
+ # TODO: combine requests and responses
95
+ # Disabled until we have a way to combine request and response objects (otherwise
96
+ # the requests are filters out but the responses are not).
97
+ #return if @options[:filter] && !(data_object.raw_data.to_s =~ /#{@options[:filter]}/i)
98
+ #return if @options[:domain] && !(data_object.uri.to_s =~ /(www.)?#{@options[:domain]}/i)
99
+
100
+ if @stella_options.format && data_object.respond_to?("to_#{@stella_options.format}")
101
+ Stella::LOGGER.info(data_object.send("to_#{@stella_options.format}"))
102
+
103
+ if data_object.has_response?
104
+ Stella::LOGGER.info(data_object.response.send("to_#{@stella_options.format}"))
105
+ end
106
+
107
+ else
108
+ if @stella_options.verbose > 1
109
+ Stella::LOGGER.info(data_object.inspect, '')
110
+
111
+ if data_object.has_response?
112
+ Stella::LOGGER.info(data_object.response.inspect, '', '')
113
+ end
114
+
115
+ elsif @stella_options.verbose > 0
116
+ Stella::LOGGER.info(data_object.to_s)
117
+ Stella::LOGGER.info(data_object.body) if data_object.has_body?
118
+
119
+ if data_object.has_response?
120
+ Stella::LOGGER.info(data_object.response.to_s)
121
+ Stella::LOGGER.info(data_object.response.body) if data_object.response.has_body?
122
+ end
123
+
124
+ else
125
+ Stella::LOGGER.info(data_object.to_s)
126
+ Stella::LOGGER.info(data_object.response.to_s) if data_object.has_response?
127
+
128
+ end
129
+ end
130
+
131
+ rescue Exception => ex
132
+ Stella::LOGGER.error(ex)
133
+ #exit 1
134
+ end
135
+
136
+
137
+
138
+ # can_pcap?
139
+ #
140
+ # Returns true is pcap is available
141
+ def can_pcap?(usepcap=false)
142
+ return false unless usepcap
143
+ begin
144
+ check_pcap
145
+ rescue
146
+ return false
147
+ end
148
+ return true
149
+ end
150
+
151
+ # check_pcap
152
+ #
153
+ # The Pcap gauntlet. A number of conditions must be met to run the Pcap recorder:
154
+ # - The watch command must be called with -p
155
+ # - The OS must be a form of Unix
156
+ # - Cannot be running via JRuby or Java
157
+ # - The user must be root (or running through sudo)
158
+ # - The Ruby-Pcap library must be installed.
159
+ def check_pcap(usepcap=false)
160
+ raise MissingDependency.new('pcap', :error_sysinfo_notunix) unless Stella::SYSINFO.os === :unix # pcap not available on windows or java
161
+ raise MissingDependency.new('pcap', :error_sysinfo_notroot) unless ENV['USER'] === 'root' # Must run as root or sudo
162
+ begin
163
+ require 'pcap'
164
+ rescue Exception, LoadError => ex
165
+ raise MissingDependency.new('pcap', :error_watch_norubypcap)
166
+ end
167
+ false
168
+ end
169
+
170
+ def generate_record_filepath
171
+ filepath = nil
172
+
173
+ if (@options[:record].is_a? String)
174
+ filepath = File.expand_path(@options[:record])
175
+ else
176
+ now = DateTime.now
177
+ daystr = "#{now.year}-#{now.mon.to_s.rjust(2,'0')}-#{now.mday.to_s.rjust(2,'0')}"
178
+ dirpath = File.join(@working_directory, 'stories', daystr)
179
+
180
+ FileUtil.create_dir(dirpath, ".")
181
+ filepath = File.join(dirpath, 'story')
182
+ testnum = 1.to_s.rjust(2,'0')
183
+ testnum.succ! while(File.exists? "#{filepath}-#{testnum}.txt")
184
+ filepath = "#{filepath}-#{testnum}.txt"
185
+ end
186
+
187
+ return filepath
188
+ end
189
+
190
+ def after
191
+ # Close Pcap / Shutdown Proxy
192
+ @watcher.after
193
+
194
+ # We don't need to close or delete a file that wasn't created
195
+ return unless @record_file
196
+
197
+ # And we don't want to delete a file that we're overwriting but may
198
+ # not have actually written anything to yet. IOW, original file will
199
+ # remain intact if we haven't written anything to it yet.
200
+ @record_file.close if @forced_overwrite
201
+
202
+ # Delete an empty file, otherwise close it
203
+ @record_file.stat.size == 0 ? File.unlink(@record_file.path) : @record_file.close
204
+ end
205
+
206
+ def process_arguments(arguments, display=false)
207
+ opts = OptionParser.new
208
+
209
+ opts.banner = "Usage: #{File.basename($0)} [global options] watch [command options] [http|dns]"
210
+ opts.on("#{$/}Example: #{File.basename($0)} -v watch -C http#{$/}")
211
+ opts.on('-h', '--help', "Display this message") do
212
+ Stella::LOGGER.info opts
213
+ exit 0
214
+ end
215
+
216
+ opts.on("#{$/}Operating mode")
217
+ opts.on('-P', '--useproxy', "Use an HTTP proxy to filter requests (default)") do |v| v end
218
+ opts.on('-C', '--usepcap', "Use Pcap to filter TCP packets") do |v| v end
219
+ #opts.on('-F', '--usepacketfu', "Use Packetfu to filter TCP packets") do |v| v end
220
+
221
+ opts.on("#{$/}Pcap-specific options")
222
+ opts.on('-i=S', '--interface=S', String, "Network device. eri0, en1, etc. (with --usepcap only)") do |v| v end
223
+ opts.on('-m=N', '--maxpacks=N', Integer, "Maximum number of packets to sniff (with --usepcap only)") do |v| v end
224
+ opts.on('-R=S', '--protocol=S', String, "Communication protocol to sniff. udp or tcp (with --usepcap only)") do |v| v end
225
+
226
+ opts.on("#{$/}Common options")
227
+ opts.on('-p=N', '--port=N', Integer, "With --useproxy this is the Proxy port. With --usecap this is the TCP port to filter. ") do |v| v end
228
+ #opts.on('-f=S', '--filter=S', String, "Filter out requests which do not contain this string") do |v| v end
229
+ #opts.on('-d=S', '--domain=S', String, "Only display requests to the given domain") do |v| v end
230
+ #opts.on('-r=S', '--record=S', String, "Record requests to file with an optional filename") do |v| v || true end
231
+ #opts.on('-F=S', '--format=S', "Format of recorded file. One of: simple (for Siege), session (for Httperf)") do |v| v end
232
+
233
+ options = opts.getopts(@arguments)
234
+ options = options.keys.inject({}) do |hash, key|
235
+ hash[key.to_sym] = options[key]
236
+ hash
237
+ end
238
+
239
+ # "interface" is more clear on the command line but we use "device" internally
240
+ options[:device] = options.delete(:interface) if options[:interface]
241
+ options[:service] = arguments.shift unless arguments.empty?
242
+
243
+ options
244
+ end
245
+
246
+ end
247
+
248
+ @@commands['watch'] = Stella::CLI::Watch
249
+ end
250
+ end
251
+
252
+
253
+ __END__
254
+
255
+ # pageload?
256
+ #
257
+ # Used while writing the session log file. Returns true when we
258
+ # suspect a new page has loaded. Otherwise the resource is considered
259
+ # to be a dependency.
260
+ def pageload?(now, think_time, host, referer, content_type)
261
+ time_difference = (now.to_i - @think_time.to_i)
262
+ time_passed = (@think_time == 0 || time_difference > 4)
263
+ non_html = (content_type !~ /text\/html/i) if content_type
264
+ #puts "POOO: #{content_type} #{referer}"
265
+
266
+ case [time_passed, non_html]
267
+ when [true,false]
268
+ return true
269
+ when [true,true]
270
+ return false
271
+ when [true,nil]
272
+ return true
273
+ when [false,false]
274
+ return false
275
+ else
276
+ return false
277
+ end
278
+ end