stella 0.5.3 → 0.5.4

data/lib/stella/adapter/pcap_watcher.rb

@@ -0,0 +1,221 @@
+
+require 'webrick'  
+require 'stringio'
+require 'net/dns/packet'
+
+
+module Stella
+  module Adapter
+    # Make sure Stella's lib directory is before the system defined ones. 
+    # We are using a modified version of pcaplet.rb. 
+    require 'pcap'
+    require 'observer'
+    
+    # Stella::Adapter::PcapWatcher
+    #
+    # Record HTTP or DNS events with Pcap (TCP sniffer). This requires ruby-pcap and the C pcap 
+    # library as well as root acceess (TCP packet sniffing requires root privileges). If you're
+    # running Ruby 1.9, JRuby, or Windows this will not be available on your system. 
+    # To sniff traffic, you must be on either the machine sending the requests or the machine
+    # receiving the requests. 
+    class PcapWatcher
+      include Observable
+      
+      # Building Ruby::Pcap with Ruby 1.9.1
+      # RSTRING()->len ia now RSTRING_LEN(), ... 
+      #   see: http://gnufied.org/2007/12/21/mysql-c-bindings-for-ruby-19/#comment-3133
+      #   see: http://www.rubyinside.com/ruby-1-9-1-preview-released-why-its-a-big-deal-1280.html#comment-37223
+      # TRAP_BEG and TRAP_END are also fucked. But the fix is not clear. 
+      # Basically Ruby::PCap is not ready for 1.9
+      # See: http://d.hatena.ne.jp/takuma104/20080210/1202638583
+      
+        # Network interface device ID. eri0, en0, lo0, etc... /sbin/ifconfig -a will tell you. 
+      attr_accessor :device
+        # Buffer size, in bytes, to read from each packet (default: 1500)
+      attr_accessor :snaplen
+        # Port of the machine sending requests (default: 80)
+      attr_accessor :sport
+        # Port of the machine receiving requests (default: 80)
+      attr_accessor :dport
+        # dns or http
+      attr_accessor :service
+        # udp or tcp
+      attr_accessor :protocol
+        # Maximum number of packets to sniff
+      attr_accessor :maxpacks
+      
+      attr_reader :pcaplet
+      
+      def initialize(options={})
+        # The proper service name for dns is "domain"
+        @service = options[:service] || 'http'
+        @service = 'domain' if options[:service] == 'dns'
+        
+        if @service == 'domain'
+          @protocol = 'udp'
+        else
+          @protocol = options[:protocol] || 'tcp'
+        end
+        
+        @dport = options[:port] || Socket::getservbyname(@service)
+        @sport = options[:port] || @dport
+        
+        @device = options[:device] || guess_device
+        @snaplen = options[:snaplen] || 1500         # 10KB
+        @maxpacks = options[:maxpacks] || 100_000
+        
+        Stella::LOGGER.info("Watching interface #{@device} for #{@service} activity on #{@protocol} port #{@dport}")
+      end
+      
+      def guess_device
+        # NOTE: This should be passed in as a value, not called from the global
+        case Stella::SYSINFO.implementation
+        when :osx
+          "en1" # Pcap.lookupdev returns en0
+        else
+          Pcap.lookupdev
+        end
+      end
+      
+      def run
+        
+        if (respond_to? "monitor_#{@service}")
+          self.send("monitor_#{@service}")
+        else
+          raise "Unknown service type (#{@service})"
+        end
+        
+      end
+      
+      
+      
+      # monitor_domain
+      #
+      # Use Ruby-Pcap to sniff packets off the network interface. 
+      #
+      # DNS monitor based on: http://www.linuxjournal.com/article/9614
+      # Install http://rubyforge.org/projects/net-dns
+      #
+      # NOTE: Is there a better way to match up a request packet with a
+      # response packet?
+      # We keep connect a request with the response using the domain name. 
+      # It's possible that two (or more) requests to be made for the same domain 
+      # at the same time and the responses could be mixed up. This will affect
+      # the exact response time but probably not by much.
+      def monitor_domain
+
+        @pcaplet = Pcaplet.new(:device => @device, :count => @maxpacks)
+        
+        req_filter  = Pcap::Filter.new("#{@protocol} and dst port #{@dport}", @pcaplet.capture)
+        resp_filter = Pcap::Filter.new("#{@protocol} and src port #{@dport}", @pcaplet.capture)
+        @pcaplet.add_filter(req_filter | resp_filter)
+        @pcaplet.each_packet do |packet|
+          data = packet.udp_data
+          case packet
+          when req_filter
+            dobj = Stella::Data::DomainRequest.new(data)
+            dobj.time = packet.time
+            dobj.client_ip = packet.ip_src
+            dobj.server_ip = packet.ip_dst
+            
+            changed
+            notify_observers(:domain_request, dobj)
+            
+          when resp_filter
+            dobj = Stella::Data::DomainResponse.new(data)
+            dobj.time = packet.time
+            dobj.client_ip = packet.ip_dst
+            dobj.server_ip = packet.ip_src
+            
+            changed
+            notify_observers(:domain_response, dobj)
+          end
+          
+        end
+
+      rescue Interrupt
+        after
+        exit
+      end
+      
+
+      def monitor_http
+        
+        @pcaplet = Pcaplet.new(:device => @device, :count => @maxpacks)
+            
+        begin
+          req_filter  = Pcap::Filter.new("#{@protocol} and dst port #{@dport}", @pcaplet.capture)
+          resp_filter = Pcap::Filter.new("#{@protocol} and src port #{@sport}", @pcaplet.capture)
+          @pcaplet.add_filter(req_filter | resp_filter)
+          @pcaplet.each_packet do |packet|
+            data = packet.tcp_data
+            next if data.nil?
+            
+            
+            # NOTE: With HTTP 1.1 keep alive connections, multiple requests can be passed
+            # through single connection. This makes it difficult to match responses with
+            # requests. 
+            # NOTE: We don't parse the body of POST and PUT requests because the data can
+            # be (and likely is), split across numerous packets. We also only grab 1500
+            # bytes from each packet. 
+            # NOTE: The hostname is taken from the Host header. Requests made without 
+            # this header (including HTTP 1.0) will contain the local hostname instead. 
+            # TODO: resolve the hostname from the IP address.   
+            # There are some helpful methods for doing some of this stuff:
+            # http://www.goto.info.waseda.ac.jp/~fukusima/ruby/pcap/doc/TCPPacket.html
+            case packet
+            when req_filter
+              next unless data and data =~ /^(GET|POST|HEAD|DELETE|PUT)\s+(.+?)\s+(HTTP.+?)$/
+              dobj = Stella::Data::HTTPRequest.new(data.gsub(/\r?\n/, $/)) # Use the system's line terminators
+              dobj.time = packet.time
+              dobj.client_ip = packet.ip_src
+              dobj.server_ip = packet.ip_dst
+              
+              changed
+              notify_observers(:http_request, dobj)  
+              
+            when resp_filter
+              # NOTE: Some responses do not contain a body in the first packet.
+              # TODO: investigate further. Try: http://www.ruby-doc.org/core/classes/Enumerable.html
+              next unless data and data =~ /^(HTTP.+)$/
+              dobj = Stella::Data::HTTPResponse.new(data.gsub(/\r?\n/, $/))
+              dobj.time = packet.time
+              dobj.client_ip = packet.ip_dst
+              dobj.server_ip = packet.ip_src
+              
+              changed
+              notify_observers(:http_response, dobj)
+                
+            end
+          
+          end
+        rescue Interrupt
+          after
+          exit
+        rescue => ex
+          Stella::LOGGER.error(ex)
+        end
+        
+        
+      end
+      
+      def after
+      STDOUT.flush
+        stat = @pcaplet.capture.stats
+        if stat
+          Stella::LOGGER.info("#{$/}#{stat.recv} packets received by filter");
+          Stella::LOGGER.info("#{stat.drop} packets dropped by kernel", ''); # with an extra line
+        end
+        STDOUT.flush
+        @pcaplet.capture.close
+        delete_observers
+      rescue
+        
+        # Ignore errors
+      end
+      
+      
+    end
+  end
+end
+

data/lib/stella/adapter/proxy_watcher.rb

@@ -0,0 +1,76 @@
+
+require 'webrick/httpproxy'
+require 'observer'
+
+
+module Stella
+  module Adapter
+    
+    # Stella::Adapter::ProxyWatcher
+    #
+    # Starts up an HTTP proxy using WEBrick to record HTTP events. This is used
+    # when PcapRecorder is not available. 
+    class ProxyWatcher
+      include Observable
+      
+      attr_accessor :port
+      
+      def initialize(options={})
+        @port = options[:port]
+      end
+      require 'pp'
+      def run
+
+        @server = WEBrick::HTTPProxyServer.new(
+            :Port => @port || 3114,
+            :AccessLog => [],
+            :ProxyContentHandler => Proc.new do |req,res|
+              
+                begin
+                  res_obj = Stella::Data::HTTPResponse.new(res.to_s)
+                  res_obj.time = Time.now
+                  
+                  req_obj = Stella::Data::HTTPRequest.new(req.to_s)
+                  req_obj.time = Time.now
+                  req_obj.client_ip = '0.0.0.0'
+                  req_obj.server_ip = '0.0.0.0'
+
+                  req_obj.response = res_obj
+
+                  changed
+                  notify_observers(:http_request, req_obj)
+                  
+                rescue SystemExit => ex
+                  after
+                  
+                rescue Exception => ex
+                  # There are miscellaneous errors (mostly to do with
+                  # incorrect content-length) that we don't care about. 
+                  Stella::LOGGER.error(ex.message)
+                end
+                
+            end
+        )
+        
+        # We need to trap this INT to kill WEBrick. Unlike with Pcap, 
+        # rescuing Interrupt doesn't work. 
+        
+        trap('INT') do
+		      after
+	      end
+	      
+        @server.start
+	      after
+	      
+      rescue => ex  
+	      after
+      end
+      
+      def after
+        delete_observers
+        @server.shutdown
+      end
+      
+    end
+  end
+end

data/lib/stella/adapter/siege.rb

@@ -32,27 +32,37 @@ module Stella
     #   -A, --user-agent="text" Sets User-Agent in request
     class Siege < Stella::Adapter::Base
       
-      
-      attr_accessor :version, :help, :config, :verbose, :get, :log, :mark, :delay, :header, :user_agent
-      attr_accessor :reps, :concurrent, :rc, :file, :time, :benchmark, :internet
+      attr_writer :reps, :concurrent, :version
+      attr_reader :user_agent
+      attr_accessor :help, :config, :verbose, :get, :log, :mark, :delay, :header
+      attr_accessor :rc, :file, :time, :benchmark, :internet
       
       def initialize(options={}, arguments=[])
         super(options, arguments)
         @name = 'siege'
         @reps = 1
         @concurrent = 1
-        @rc = File.join(ENV['HOME'], '.siegerc')
         @private_variables = ['private_variables', 'name', 'arguments', 'load_factor', 'working_directory', 'orig_logfile']
         @load_factor = 1
+        
+        @rc = File.join(ENV['HOME'] || ENV['USERPROFILE'], '.siegerc')
+        
+        # Siege won't run unless there's a siegerc file. If the default one doesn't exist
+        # we need to call siege.config to create it. This should only happen once. 
+        # We use capture_output here so STDOUT and STDERR don't print to the screen. 
+        Stella::Util.capture_output("#{@name}.config") do 'nothing' end unless File.exists? @rc
       end
       
       
+      
+      def error
+        (File.exists? stderr_path) ? FileUtil.read_file(stderr_path) : "Unknown error"
+      end
+      
       def version
         vsn = 0
-        text = ""
-        Open3.popen3("#{@name} --version") do |stdin, stdout, stderr| 
-          text = stderr.readlines.join
-          text.scan(/SIEGE (\d+?\.\d+)/) { |v| vsn = v[0] }
+        Stella::Util.capture_output("#{@name} --version") do |stdout, stderr|
+           stderr.join.scan(/SIEGE (\d+?\.\d+)/) { |v| vsn = v[0] }
         end
         vsn
       end
@@ -137,8 +147,15 @@ module Stella
         opts.on('-i', '--internet') do |v| @internet = true; end
         opts.on('-A S', '--user-agent=S', String) do |v| @user_agent ||= []; @user_agent << v end
 
-        raise "You cannot select both --internet and --benchmark" if options.internet && options.benchmark
-
+        unless options.benchmark
+          Stella::LOGGER.warn('--benchmark (or -b) is not selected. Siege will include "think-time" for all requests.') 
+        end
+        
+        opts.on('-n N',Integer) do |v| 
+          Stella::LOGGER.error("-n is not a Siege parameter. You probably want -r.")
+          exit 1
+        end
+        
         # parse! removes the options it finds.
         # It also fails when it finds unknown switches (i.e. -X)
         # Which should leave only the remaining arguments (URIs in this case)
@@ -257,7 +274,7 @@ module Stella
 
       # Siege writes the summary to STDERR
       def stats_file
-        File.new(stderr_path)
+        File.new(stderr_path) if File.exists?(stderr_path)
       end
       
       def rc_file

data/lib/stella/cli/agents.rb

@@ -42,8 +42,8 @@ module Stella
           end
         end
  
-        puts (@list) ? agents.uniq.sort.join("\n") : Stella::TEXT.msg(:agents_count_message, agents.uniq.size)
-
+        msg = (@list) ? agents.uniq.sort.join("\n") : Stella::TEXT.msg(:agents_count_message, agents.uniq.size)
+        puts msg
       end
       
       def process_options(display=false)

data/lib/stella/cli/base.rb

@@ -13,7 +13,43 @@ module Stella
       def initialize(adapter)
         @adapter_name = adapter
         @options = OpenStruct.new
+        
+        # There is a bug in Ruby 1.8.6 where a trapped SIGINT will hang. 
+        # This workaround is from: http://redmine.ruby-lang.org/issues/show/362 
+        # It works in Ruby 1.9 and JRuby as well. 
+        # NEW WARNING: This puts the process into a new thread which somehow 
+        # prevents Pcap from reporting on UDP/DNS packets (TCP/HTTP is unaffected).
+        # I left this here as an example of how not to it. Incidentally, 
+        # "rescue Interrupt" seems to be working fine now. 
+        # See also: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/220937
+        #@killer = Thread.new do
+        #  puts "#{$/}Exiting...#{$/}"
+        #  Thread.stop
+        #  Thread.main.join(1)
+        #  exit 0
+        #end
+        #
+        # Note that this is just a default. Any class that implements another
+        # Signal.trap will override this.
+        #Signal.trap('INT') do
+        #  @killer.call
+        #  exit 0
+        #end
       end
     end
   end
-end
+end
+
+__END__
+
+TODO: Investigate frylock style definition
+include Stella::CLI::Base
+
+before do
+  # stuff that would go in initialize
+end
+
+command 'sysinfo' do
+  puts Stella::SYSINFO.to_yaml(:headers)
+end
+

data/lib/stella/cli/localtest.rb

@@ -45,7 +45,6 @@ module Stella
         @driver.working_directory = @working_directory
         
         @driver.run
-        
       end
 
 
@@ -54,7 +53,7 @@ module Stella
       # 
       # Populates @testdef with values from @stella_options
       def process_stella_options
-        @testdef.repetitions = @stella_options.testreps
+        @testdef.repetitions = @stella_options.repetitions
         @testdef.sleep = @stella_options.sleep
         @testdef.warmup = @stella_options.warmup
         @testdef.rampup = @stella_options.rampup

data/lib/stella/cli/sysinfo.rb

@@ -0,0 +1,17 @@
+
+module Stella 
+  class CLI
+    class SystemInfo < Stella::CLI::Base
+      
+      
+      def run
+        puts Stella::SYSINFO
+        #puts "HI"
+      end
+      
+    end
+    
+    @@commands['sysinfo'] = Stella::CLI::SystemInfo
+  end
+end
+