stella 0.5.3 → 0.5.4

data/lib/stella/cli/watch.rb

@@ -0,0 +1,278 @@
+
+# TODO: Record cookies. 
+# TODO: Investigate packetfu: http://code.google.com/p/packetfu/
+# TODO: Investigate Winpcap (http://www.winpcap.org/) and libpcap on Windows 
+
+module Stella 
+  class CLI
+    class Watch < Stella::CLI::Base
+      
+
+      def run
+        @options = process_arguments(@arguments)
+        
+        if can_pcap?(@options[:usepcap]) 
+          require 'stella/adapter/pcap_watcher'
+          @watcher = Stella::Adapter::PcapWatcher.new(@options)
+ 
+        else
+          require 'stella/adapter/proxy_watcher'
+          @watcher = Stella::Adapter::ProxyWatcher.new(@options)
+          
+          # if can_pcap? returned false, but pcap was requested then we'll
+          # call check_pcap to raise the reason why it didn't load. 
+          if @options[:usepcap]
+            check_pcap
+            exit 0
+          end
+        end
+        
+        # We use an observer model so the watcher class will notify us
+        # when they have new data. They call the update method below. 
+        @watcher.add_observer(self)
+        
+        if @options[:record]
+          
+          @record_filepath = generate_record_filepath 
+          Stella::LOGGER.info("Writing to #{@record_filepath}")
+        
+          if File.exists?(@record_filepath) 
+            Stella::LOGGER.error("#{@record_filepath} exists")
+            if @stella_options.force
+              Stella::LOGGER.error("But I'll overwrite it and continue because you forced me too!")
+            else
+              exit 1
+            end
+          end
+          
+        end
+        
+        Stella::LOGGER.info("Filter: #{@options[:filter]}") if @options[:filter]
+        Stella::LOGGER.info("Domain: #{@options[:domain]}") if @options[:domain]
+        
+        # Turn wildcards into regular expressions
+        @options[:filter].gsub!('*', '.*') if @options[:filter] 
+        @options[:domain].gsub!('*', '.*') if @options[:domain]
+        
+        # Used to calculated user think times for session output
+        @think_time = 0
+        
+        @watcher.run
+        
+      end
+      
+
+      
+      # update
+      #
+      # This method is called from the watcher class when data is updated. 
+      # +service+ is one of: domain, http_request, http_response
+      # +data+ is a string of TCP packet data. The format depends on the value of +service+.
+      def update(service, data_object)
+        
+        begin
+          if @options[:record] && !@file_created_already
+        
+            if File.exists?(@record_filepath) 
+              if @stella_options.force
+  		          @record_file = FileUtil.create_file(@record_filepath, 'w', '.', :force)
+              else
+                exit 1
+              end
+            else
+  		        @record_file = FileUtil.create_file(@record_filepath, 'w', '.')
+            end
+	        
+  	        raise StellaError.new("Cannot open: #{@record_filepath}") unless @record_file
+
+            @file_created_already = true
+          end
+        rescue => ex
+          raise StellaError.new("Error creating file: #{ex.message}")
+        end
+        
+        # TODO: combine requests and responses
+        # Disabled until we have a way to combine request and response objects (otherwise
+        # the requests are filters out but the responses are not).
+        #return if @options[:filter] && !(data_object.raw_data.to_s =~ /#{@options[:filter]}/i)
+        #return if @options[:domain] && !(data_object.uri.to_s =~ /(www.)?#{@options[:domain]}/i)
+        
+        if @stella_options.format && data_object.respond_to?("to_#{@stella_options.format}")
+          Stella::LOGGER.info(data_object.send("to_#{@stella_options.format}"))
+          
+          if data_object.has_response?
+            Stella::LOGGER.info(data_object.response.send("to_#{@stella_options.format}"))
+          end
+          
+        else 
+          if @stella_options.verbose > 1
+            Stella::LOGGER.info(data_object.inspect, '')
+            
+            if data_object.has_response?
+              Stella::LOGGER.info(data_object.response.inspect, '', '')
+            end
+            
+          elsif @stella_options.verbose > 0 
+            Stella::LOGGER.info(data_object.to_s)
+            Stella::LOGGER.info(data_object.body) if data_object.has_body?
+            
+            if data_object.has_response?
+              Stella::LOGGER.info(data_object.response.to_s) 
+              Stella::LOGGER.info(data_object.response.body) if data_object.response.has_body?
+            end
+            
+          else
+            Stella::LOGGER.info(data_object.to_s)
+            Stella::LOGGER.info(data_object.response.to_s) if data_object.has_response?
+            
+          end
+        end
+                
+      rescue Exception => ex
+        Stella::LOGGER.error(ex)
+        #exit 1
+      end
+      
+      
+      
+      # can_pcap?
+      #
+      # Returns true is pcap is available 
+      def can_pcap?(usepcap=false)
+        return false unless usepcap
+        begin
+          check_pcap
+        rescue
+          return false
+        end
+        return true
+      end
+      
+      # check_pcap
+      #
+      # The Pcap gauntlet. A number of conditions must be met to run the Pcap recorder:
+      # - The watch command must be called with -p
+      # - The OS must be a form of Unix
+      # - Cannot be running via JRuby or Java
+      # - The user must be root (or running through sudo)
+      # - The Ruby-Pcap library must be installed.
+      def check_pcap(usepcap=false)
+        raise MissingDependency.new('pcap', :error_sysinfo_notunix) unless Stella::SYSINFO.os === :unix # pcap not available on windows or java
+        raise MissingDependency.new('pcap', :error_sysinfo_notroot) unless ENV['USER'] === 'root' # Must run as root or sudo
+        begin 
+          require 'pcap'
+        rescue Exception, LoadError => ex
+          raise MissingDependency.new('pcap', :error_watch_norubypcap)
+        end
+        false
+      end
+      
+      def generate_record_filepath
+        filepath = nil
+        
+        if (@options[:record].is_a? String)
+          filepath = File.expand_path(@options[:record])
+        else
+          now = DateTime.now
+          daystr = "#{now.year}-#{now.mon.to_s.rjust(2,'0')}-#{now.mday.to_s.rjust(2,'0')}"
+          dirpath = File.join(@working_directory, 'stories', daystr)
+
+          FileUtil.create_dir(dirpath, ".")
+          filepath = File.join(dirpath, 'story')
+          testnum = 1.to_s.rjust(2,'0')
+          testnum.succ! while(File.exists? "#{filepath}-#{testnum}.txt")
+          filepath = "#{filepath}-#{testnum}.txt"
+        end
+          
+        return filepath
+      end
+      
+      def after
+        # Close Pcap / Shutdown Proxy
+        @watcher.after
+        
+        # We don't need to close or delete a file that wasn't created
+        return unless @record_file
+        
+        # And we don't want to delete a file that we're overwriting but may
+        # not have actually written anything to yet. IOW, original file will
+        # remain intact if we haven't written anything to it yet. 
+        @record_file.close if @forced_overwrite
+        
+        # Delete an empty file, otherwise close it
+        @record_file.stat.size == 0 ? File.unlink(@record_file.path) : @record_file.close
+      end
+      
+      def process_arguments(arguments, display=false)
+        opts = OptionParser.new
+        
+        opts.banner = "Usage: #{File.basename($0)} [global options] watch [command options] [http|dns]"
+        opts.on("#{$/}Example: #{File.basename($0)} -v watch -C http#{$/}")
+        opts.on('-h', '--help', "Display this message") do
+          Stella::LOGGER.info opts
+          exit 0
+        end
+        
+        opts.on("#{$/}Operating mode")
+        opts.on('-P', '--useproxy', "Use an HTTP proxy to filter requests (default)") do |v| v end
+        opts.on('-C', '--usepcap', "Use Pcap to filter TCP packets") do |v| v end
+        #opts.on('-F', '--usepacketfu', "Use Packetfu to filter TCP packets") do |v| v end
+          
+        opts.on("#{$/}Pcap-specific options")
+        opts.on('-i=S', '--interface=S', String, "Network device. eri0, en1, etc. (with --usepcap only)") do |v| v end
+        opts.on('-m=N', '--maxpacks=N', Integer, "Maximum number of packets to sniff (with --usepcap only)") do |v| v end
+        opts.on('-R=S', '--protocol=S', String, "Communication protocol to sniff. udp or tcp (with --usepcap only)") do |v| v end
+        
+        opts.on("#{$/}Common options")
+        opts.on('-p=N', '--port=N', Integer, "With --useproxy this is the Proxy port. With --usecap this is the TCP port to filter. ") do |v| v end
+        #opts.on('-f=S', '--filter=S', String, "Filter out requests which do not contain this string") do |v| v end
+        #opts.on('-d=S', '--domain=S', String, "Only display requests to the given domain") do |v| v end
+        #opts.on('-r=S', '--record=S', String, "Record requests to file with an optional filename") do |v| v || true end
+        #opts.on('-F=S', '--format=S', "Format of recorded file. One of: simple (for Siege), session (for Httperf)") do |v| v end
+            
+        options = opts.getopts(@arguments)  
+        options = options.keys.inject({}) do |hash, key|
+           hash[key.to_sym] = options[key]
+           hash
+        end
+                
+        # "interface" is more clear on the command line but we use "device" internally
+        options[:device] = options.delete(:interface) if options[:interface]
+        options[:service] = arguments.shift unless arguments.empty?
+        
+        options
+      end
+      
+    end
+    
+    @@commands['watch'] = Stella::CLI::Watch
+  end
+end
+
+
+__END__
+
+# pageload?
+#
+# Used while writing the session log file. Returns true when we
+# suspect a new page has loaded. Otherwise the resource is considered 
+# to be a dependency. 
+def pageload?(now, think_time, host, referer, content_type)
+  time_difference = (now.to_i - @think_time.to_i)
+  time_passed = (@think_time == 0 || time_difference > 4) 
+  non_html = (content_type !~ /text\/html/i) if content_type
+  #puts "POOO: #{content_type} #{referer}"
+  
+  case [time_passed, non_html]
+  when [true,false]
+    return true
+  when [true,true]
+    return false
+  when [true,nil]
+    return true
+  when [false,false]
+    return false
+  else
+    return false
+  end
+end

data/lib/stella/cli.rb

@@ -1,4 +1,6 @@
 
+# http://www.ruby-doc.org/stdlib/libdoc/observer/rdoc/index.html
+
 require 'optparse'
 require 'ostruct'
 
@@ -10,8 +12,11 @@ module Stella
   #
   # This Config class manages the content of ENV['HOME]/.stella. The functionality 
   # is currently disabled so stella is stateless. 
+  # RUBY_PLATFORM = 'java' in jruby and i386-mswin32 for windows
   class Config < Storable
-    DEFAULT_HOME = File.join(ENV['HOME'], '.stella').freeze unless defined? DEFAULT_HOME
+    USER_HOME = ENV['USERPROFILE'] || ENV['HOME']
+    STELLA_DIR = '.stella'
+    DEFAULT_HOME = File.join(USER_HOME, STELLA_DIR).freeze unless defined? DEFAULT_HOME
     DEFAULT_DATA_HOME = File.join(Dir.getwd, 'stella').freeze unless defined? DEFAULT_DATA_HOME
     
     attr_accessor :conf_path, :data_path
@@ -69,6 +74,8 @@ module Stella
       @options.data_path = @config.working_directory
       @options.agents = []
       
+      @stella_arguments = []
+      @command_arguments = []
     end
     
     def commands
@@ -76,6 +83,7 @@ module Stella
     end
     
     def run
+      
       process_arguments
       process_options
       
@@ -89,14 +97,12 @@ module Stella
       command = @@commands[@command_name].new(@command_name)
       
       # Give the command object access to the config and runtime options
-      #command.global_config = @config 
       command.stella_options = @options
       command.arguments = @command_arguments
       command.working_directory = @options.data_path
       
       command.run
       
-
     rescue => ex
       Stella::LOGGER.error(ex)
     end
@@ -122,9 +128,9 @@ module Stella
               break
             end
           end
-
-          @command_arguments = [] if @command_arguments.nil? 
-          @stella_arguments = [] if @stella_arguments.nil?
+          
+          @stella_arguments = [] unless @stella_arguments
+          @command_arguments = [] unless @command_arguments 
 
           # If there's no command we'll assume all the options are for Stella
           unless @command_name
@@ -143,7 +149,7 @@ module Stella
 
           opts = OptionParser.new 
           opts.banner = Stella::TEXT.msg(:option_help_usage)
-          opts.on Stella::TEXT.msg(:option_help_preamble, @@commands.keys.join(', '))
+          opts.on Stella::TEXT.msg(:option_help_preamble, @@commands.keys.sort.join(', '))
 
           opts.on(Stella::TEXT.msg(:option_help_options_title))
           opts.on('-V', '--version', Stella::TEXT.msg(:option_help_version)) do
@@ -151,8 +157,10 @@ module Stella
             exit 0
           end
           opts.on('-h', '--help', Stella::TEXT.msg(:option_help_help)) { puts opts; exit 0 }
+          opts.on('-F', '--force', Stella::TEXT.msg(:option_help_force)) { |v| @options.force = true }
 
           opts.on('-v', '--verbose', Stella::TEXT.msg(:option_help_verbose)) do
+            
             @options.verbose ||= 0
             @options.verbose += 1 
           end
@@ -189,8 +197,8 @@ module Stella
             @options.rampup = amount
           end
 
-          opts.on('-x', '--testreps=N', Integer, Stella::TEXT.msg(:option_help_testreps)) do |v| 
-            @options.testreps = MathUtil.enforce_limit(v,1,10)
+          opts.on('-x', '--repetitions=N', Integer, Stella::TEXT.msg(:option_help_testreps)) do |v| 
+            @options.repetitions = MathUtil.enforce_limit(v,1,99)
           end
 
           opts.on('-w [N]', '--warmup', Float, Stella::TEXT.msg(:option_help_warmup, 0.1)) do |v|
@@ -282,10 +290,14 @@ end
 
 # Autoload CLI classes. These classes add themselves to the class variable @@commands. 
 begin
+  previous_path = ""
   cli_classes = Dir.glob(File.join(STELLA_HOME, 'lib', 'stella', 'cli', "*.rb"))
   cli_classes.each do |path|
+    previous_path = path
     require path
   end
 rescue LoadError => ex
-  Stella::LOGGER.info("Error loading #{path}: #{ex.message}")
-end
+  Stella::LOGGER.info("Error loading #{previous_path}: #{ex.message}")
+end
+
+

data/lib/stella/command/base.rb

@@ -1,5 +1,5 @@
 
-
+      
 module Stella::Command
   class Base
     
@@ -98,14 +98,5 @@ module Stella::Command
     end
 
 
-    # 
-    # Generates a string of random alphanumeric characters
-    # These are used as IDs throughout the system
-    def strand( len )
-       chars = ("a".."z").to_a + ("0".."9").to_a
-       newpass = ""
-       1.upto(len) { |i| newpass << chars[rand(chars.size-1)] }
-       return newpass
-    end
   end
 end

data/lib/stella/command/localtest.rb

@@ -2,9 +2,7 @@
 
 module Stella
   class LocalTest < Stella::Command::Base
-    
-    
-    
+
       # A container for the test parameters
     attr_accessor :testdef
       # The load tool adapter
@@ -33,7 +31,8 @@ module Stella
       @testdef = testdef if testdef
       @adapter = adapter if adapter
       
-      @guid = Crypto.sign(rand.to_s, rand.to_s)
+      # Disabled until we resolve JRuby on OSX issue (won't load openssl)
+      #@guid = Crypto.sign(rand.to_s, rand.to_s)
       
       @test_runpaths = []       
       @all_runpaths = []        
@@ -64,9 +63,9 @@ module Stella
       
       raise UnavailableAdapter.new(@adapter.name) unless @adapter.available?
       
+      # If the adapter isn't being called for a loadtest, we don't have anything to do.
       unless @adapter.loadtest?
         system(@adapter.command)
-
         return
       end
             
@@ -157,6 +156,11 @@ module Stella
         save_summary(File.join(test_path, "SUMMARY-RAMPUP.#{@format}"), test_stats)
         print_summary(test_stats) if (@testdef.repetitions > 1)
       end
+    rescue Interrupt
+      puts "HIHIHI2222"
+      exit
+    rescue AdapterError => ex
+      Stella::LOGGER.error(ex.message)
     end
     
     def latest_test_symlink_path
@@ -181,11 +185,12 @@ module Stella
         # Make sure the test storage directory is created along with the
         # latest symlink
         FileUtil.create_dir(test_path)
-        File.unlink(latest_test_symlink_path) if File.exists?(latest_test_symlink_path)
-        File.symlink(File.expand_path(test_path), latest_test_symlink_path)
+        #File.unlink(latest_test_symlink_path) if File.exists?(latest_test_symlink_path)
+        #File.symlink(File.expand_path(test_path), latest_test_symlink_path)
         
         # Write the test ID to the storage directory
-        FileUtil.write_file(test_path + "/ID.txt", @guid, true)
+        # NOTE: Disabled until we resolve the issue with JRuby on OSX (won't load openssl)
+        #FileUtil.write_file(test_path + "/ID.txt", @guid, true)
 
         # And the test run message
         FileUtil.write_file(test_path + "/MESSAGE.txt", @testdef.message, true) if @testdef.message
@@ -241,25 +246,37 @@ module Stella
           FileUtil.write_file(runpath + "/COMMAND.txt", @adapter.command, true)
         
           # Execute the command, send STDOUT and STDERR to separate files. 
-          command = "#{@adapter.command} 1> '#{@adapter.stdout_path}' 2> '#{@adapter.stderr_path}'"
+          command = "#{@adapter.command} 1> \"#{@adapter.stdout_path}\" 2> \"#{@adapter.stderr_path}\""
           Stella::LOGGER.info(" COMMAND: #{command}") if @verbose >= 2
-        
-          # Call the load tool
-          # for Windows, see: http://blog.crankybit.com/redirecting-output-to-a-file-in-windows-batch-scripts/
-          #                   http://weblogs.asp.net/lorenh/archive/2006/03/24/441004.aspx
-          system(command)
-        
+          
+          begin
+            # Call the load tool
+            # $? contains the error status
+            succeeded = system(command)  
+          
+          # TODO: Catch interrupts for system calls. Currently it will simply and and continue with the next command
+          # i.e. these don't work:
+          rescue Interrupt
+            exit
+          rescue SystemExit
+            exit
+          end
+          
+          unless succeeded
+            Stella::LOGGER.info('', '') # We used print so we need a new line for the error message.
+            raise AdapterError.new(@adapter.name, @adapter.error) 
+          end
+          
           stats = @adapter.stats
       
           save_summary(@adapter.summary_path(@format), stats)
       
 
-          unless @quiet
-            Stella::LOGGER.info_print(sprintf("%3.0f%% %9.2f/s %8.3fs ", stats.availability, stats.transaction_rate, stats.response_time))
-            Stella::LOGGER.info_print(sprintf("%8.3fMB/s %8.3fMB %8.3fs  ", stats.throughput, stats.data_transferred, stats.elapsed_time))
+          if !@quiet && stats && stats.available?
+            Stella::LOGGER.info_print(sprintf("%3.0f%% %9.2f/s %8.3fs ", stats.availability || 0, stats.transaction_rate || 0, stats.response_time || 0))
+            Stella::LOGGER.info_print(sprintf("%8.3fMB/s %8.3fMB %8.3fs  ", stats.throughput || 0, stats.data_transferred || 0, stats.elapsed_time || 0))
             # NOTE: We don't print a line terminator here
           end
-          
         end
       end
     
@@ -274,10 +291,10 @@ module Stella
       def print_summary(stats)
         Stella::LOGGER.info(' ' << "-"*67) unless @quiet
 
-        Stella::LOGGER.info_printf("%8s: %10d@%-6d% 3.0f%% %9.2f/s ", "Total", stats.transactions_total, stats.vusers_avg, stats.availability, stats.transaction_rate_avg)
-        Stella::LOGGER.info_printf("%8.3fs %8.3fMB/s %8.3fMB %8.3fs", stats.response_time_avg, stats.throughput_avg, stats.data_transferred_total, stats.elapsed_time_total)
+        Stella::LOGGER.info_printf("%8s: %10d@%-6d %3.0f%% %9.2f/s ", "Total", stats.transactions_total || 0, stats.vusers_avg || 0, stats.availability || 0, stats.transaction_rate_avg || 0)
+        Stella::LOGGER.info_printf("%8.3fs %8.3fMB/s %8.3fMB %8.3fs", stats.response_time_avg || 0, stats.throughput_avg || 0, stats.data_transferred_total || 0, stats.elapsed_time_total || 0)
         Stella::LOGGER.info('') # New line
-        Stella::LOGGER.info_printf("%8s: %22s %9.2f/s %8.3fs %8.3fMB/s %10s %8.3fs", "Std Dev", '', stats.transaction_rate_sdev, stats.response_time_sdev, stats.throughput_sdev, '', stats.elapsed_time_sdev)
+        Stella::LOGGER.info_printf("%8s: %22s %9.2f/s %8.3fs %8.3fMB/s %10s %8.3fs", "Std Dev", '', stats.transaction_rate_sdev || 0, stats.response_time_sdev || 0, stats.throughput_sdev || 0, '', stats.elapsed_time_sdev || 0)
         Stella::LOGGER.info('') # New line
         Stella::LOGGER.info('') unless @quiet # Extra new line
       end
@@ -303,16 +320,19 @@ module Stella
       # filepath:: the complete path for the file (string)
       # stats:: Any object that extends Stella::Test::Base object
       def save_summary(filepath, stats)
-        FileUtil.write_file(filepath, stats.dump(@format, :with_titles), true)
+        FileUtil.write_file(filepath, stats.dump(@format, :with_titles), true) if stats
       end
       
       # Load SUMMARY file for each run and create a summary with
       # totals, averages, and standard deviations. 
       def process_test_stats(paths)
+        return unless paths && !paths.empty?
         test_stats = Stella::Test::Summary.new(@message)
+        return unless test_stats
         all_stats_obj = []
         paths.each do |path|
           file_contents = FileUtil.read_file_to_array("#{path}/SUMMARY.#{@format}")
+          next if !file_contents || file_contents.empty?
           stats = Stella::Test::Run::Summary.undump(@format, file_contents)
           stats_obj = Stella::Test::Run::Summary.from_hash(stats)
           test_stats.add_run(stats_obj)

data/lib/stella/data/domain.rb

@@ -0,0 +1,75 @@
+
+
+module Stella::Data
+  class DomainRequest < Stella::Storable
+    attr_accessor :time, :client_ip, :server_ip, :dns_data, :domain_name, :header
+    attr_reader :raw_data
+ 
+    def initialize(raw_data)
+      @raw_data = raw_data
+      @dns_data, @domain_name, @header = DomainUtil::parse_domain_request(@raw_data)
+    end
+    
+    def has_request?
+      false
+    end
+    def has_response?
+      false
+    end
+    def has_body?
+      false
+    end
+    
+    def field_names
+      [ :time, :client_ip, :server_ip, :domain_name, :header ]
+    end
+    
+    def to_s
+      "%s: %s -> %s (%s)" % [@time, @client_ip, @server_ip, @domain_name]
+    end
+    
+    def inspect
+      str = "#{$/};; REQUEST         #{@time.to_s}"
+      str << "#{$/};; %s %s> %s" % [@client_ip, '-'*30, @server_ip]
+      str << "#{$/};;#{$/}"
+      str << @dns_data.inspect
+      str
+    end
+    
+  end
+  
+  class DomainResponse < Stella::Storable
+    attr_accessor :time, :client_ip, :server_ip, :dns_data, :domain_name, :header, :addresses, :cnames
+    attr_reader :raw_data
+ 
+    def initialize(raw_data)
+      @raw_data = raw_data
+      @dns_data, @domain_name, @header, @addresses, @cnames = DomainUtil::parse_domain_response(@raw_data)
+    end
+    
+    def has_request?
+      false
+    end
+    def has_response?
+      false
+    end
+    def has_body?
+      false
+    end
+    
+    def field_names
+      [ :time, :client_ip, :server_ip, :dns_data, :domain_name, :header, :addresses, :cnames ]
+    end
+    
+    def to_s
+      "%s: %s <- %s (%s) %s" % [@time, @client_ip, @server_ip, @domain_name, (@addresses || []).join(',')]
+    end
+    
+    def inspect
+      str =  "#{$/};; RESPONSE        #{@time.strftime(NICE_TIME_FORMAT)}"
+      str << "#{$/};; %s <%s %s" % [@client_ip, '-'*30, @server_ip]
+      str << "#{$/};;#{$/}"
+      str << @dns_data.inspect
+    end
+  end
+end