nethttp_ab 0.0.5 → 0.0.6

data/CHANGELOG

@@ -1,3 +1,8 @@
+v0.0.6 [09-05-2011]
+* added an ability to set a follow_links depth level to simulate a real user
+* added verbose option
+* refactored code a bit, fixed some errors
+
 v0.0.5 [15-04-2011]
 * fixed issue #1 ("HTTP request path is empty")
 * following the redirect (if any) on the first request (using Net::HTTP.get_response instead of Net::HTTP::Get) so deleted net.rb file

data/Manifest

@@ -11,6 +11,15 @@ lib/simple_requests_queue.rb
 nethttp_ab.gemspec
 test/nethttp_ab_test.rb
 test/requests_queue_test.rb
+test/resources/depth2/links1.html
+test/resources/depth2/links2.html
+test/resources/depth2/links3.html
+test/resources/depth2/links_11.html
+test/resources/depth2/links_12.html
+test/resources/depth2/links_21.html
+test/resources/depth2/links_31.html
+test/resources/depth2/links_32.html
+test/resources/depth2/links_33.html
 test/resources/index.html
 test/resources/links.html
 test/resources/links1.html
@@ -18,5 +27,6 @@ test/resources/links2.html
 test/resources/links3.html
 test/resources/links4.html
 test/resources/links5.html
+test/resources/links_depth_2.html
 test/simple_requests_queue_test.rb
 test/url_regexp_test.rb

data/README

@@ -8,9 +8,17 @@ nethttp_ab -n100 -c3 http://www.yoursite.com
 OR simulate one user and follow all local links on the page:
 nethttp_ab -f http://localhost:3000
 
-simulate 3 users (all local links on the page will be visited once)
+OR simulate a user who will follow all inner links until depth 2
+that is [main_pages_links]:depth0 => [link1, link2, link3]:depth1 => [link11, link12, link21, link22, link31]:depth2
+nethttp_ab -f2 http://localhost:3000
+
+OR simulate 3 users (all local links on the page will be visited once)
 nethttp_ab --follow_links -c3 http://localhost:3000
 
+
+Also you could add --verbose or -v option to check which urls being requested in follow_links mode.
+
+
 Issue tracker: https://github.com/german/nethttp_ab/issues
 
 

data/Rakefile

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('nethttp_ab', '0.0.5') do |p|
+Echoe.new('nethttp_ab', '0.0.6') do |p|
   p.description    = "Simple tool to test and benchmark sites"
   p.url            = "http://github.com/german/nethttp_ab"
   p.author         = "Dmitrii Samoilov"

data/bin/nethttp_ab

@@ -14,23 +14,15 @@ require File.dirname(File.expand_path(__FILE__)) + '/../lib/requester.rb'
 
 opts = {}
 
-if ARGV.include?('--follow_links') || ARGV.include?('-f')
-  opts[:follow_links] = true
-else
-  opts[:follow_links] = false
-end
+opts[:follow_links] = ARGV.include?('--follow_links') || ARGV.include?('-f')
 
-if ARGV.join.match(/-c\s*(\d*)/)
-  opts[:concurrent_threads] = $1.to_i
-else
-  opts[:concurrent_threads] = 1
-end
+opts[:concurrent_threads] = ARGV.join.match(/-c\s*(\d*)/) ? $1.to_i : 1
 
-if ARGV.join.match(/-n\s*(\d*)/)
-  opts[:requests] = $1.to_i
-else
-  opts[:requests] = 10
-end
+opts[:follow_links_depth] = ARGV.join.match(/-f\s*(\d*)/) ? $1.to_i : 0
+
+opts[:requests] = ARGV.join.match(/-n\s*(\d*)/) ? $1.to_i : 10
+
+opts[:verbose] = ARGV.include?('--verbose') || ARGV.include?('-v')
 
 url_to_benchmark = "http://localhost:3000/"
 
@@ -42,9 +34,12 @@ end
 url_to_benchmark = "http://" + url_to_benchmark if url_to_benchmark !~ /^https?:\/\//
 
 requester = NethttpAb::Requester.new
+requester.verbose           = opts[:verbose]
 requester.concurrent_users  = opts[:concurrent_threads]
 requester.num_of_requests   = opts[:requests]
 requester.follow_links      = opts[:follow_links]
+requester.follow_links_depth= opts[:follow_links_depth]
+
 requester.url               = url_to_benchmark
 
 requester.proceed

data/lib/requester.rb

@@ -25,6 +25,12 @@ module NethttpAb
       @failed_requests   = 0
       @successfull_requests  = 0
       @follow_links = false
+      @follow_links_depth = 1
+      @verbose = false
+    end
+
+    def verbose=(flag)
+      @verbose = flag
     end
 
     def concurrent_users=(num)
@@ -38,7 +44,15 @@ module NethttpAb
     def follow_links=(flag)
       @follow_links = flag
       # we could follow links on the pages if there's --follow-links=2 option
-      require 'nokogiri'  if @follow_links
+      puts 'follow_links mode' if @verbose && @follow_links
+      require 'nokogiri' if @follow_links
+    end
+
+    def follow_links_depth=(depth)
+      if depth > 0
+        self.follow_links = true # set follow_links flag to true (if cli option was -f2 for example)
+        @follow_links_depth = depth
+      end
     end
 
     def url=(link)
@@ -68,60 +82,91 @@ module NethttpAb
 
     private
 
-      def prepare_queue
-        @requests_queue = if @follow_links
-          # get all links to benchmark as user behavior
-          response = begin
-            Net::HTTP.get_response(@url)
-          rescue OpenSSL::SSL::SSLError => e
-            puts("The url you provided is wrong, please check is it really ssl encrypted")
-            exit
-          rescue Errno::ECONNREFUSED => e
-            puts("Connection error, please check your internet connection or make sure the server is running (it's local)")
-            exit
-          rescue Timeout::Error => e
-            puts("Timeout error: please check the site you're benchmarking")
-            exit          
-          rescue SocketError => e
-            puts e.message
-            exit
-          end
-          
-          response = case response
-            when Net::HTTPSuccess
-              response
-            when Net::HTTPRedirection
-              print "redirected to #{response['location']}\n"
-              # we must correct the url, so we could select right inner links then
-              # consider this: you request www.example.com and it redirects you to example.com
-              # but the @url.host will be still www.example.com, so href.match(Regexp.escape(@url.host)) later will fail
-              self.url = response['location']
-              Net::HTTP.get_response(@url)
-          end
-          
-          doc = Nokogiri::HTML(response.body)
+      def extract_links_from(doc)
+        doc.css('a').reject{|el| el.attr('rel') == 'nofollow' || el.attr('href') =~ /^javascript/ || el.attr('onclick')}
+      end
 
-          local_links = doc.css('a').reject{|el| el.attr('rel') == 'nofollow' || el.attr('href') =~ /^javascript/ || el.attr('onclick')}
+      def select_local_links_from(all_links, parent_url_parsed)
+        all_links.map!{|el| el.attr('href') }
 
-          local_links.map!{|el| el.attr('href') }
+        local_links = all_links.select{|href| href.match(Regexp.escape(parent_url_parsed.host)) || href !~ URL_REGEXP }
 
-          local_links = local_links.select{|href| href.match(Regexp.escape(@url.host)) || href !~ URL_REGEXP }
+        # we assume that local_links now contains only links inner to tested site
+        local_links.map! do |href|
+          # a proper url should be constructed which then could be parsed by URI.parse
+          if !href.match(Regexp.escape(parent_url_parsed.host))
 
-          # we assume that local_links now contains only links inner to tested site
-          local_links.map! do |href|
-            # we must construct a proper url which then could be parsed by URI.parse
-            if !href.match(Regexp.escape(@url.host))
-              href = if (href[0] == '/' && @url.host[-1] != '/') || (href[0] != '/' && @url.host[-1] == '/')
-                "#{@url.scheme}://#{@url.host}#{href}"
-              else
-                "#{@url.scheme}://#{@url.host}/#{href}"
-              end
+            # relative links should be treated respectively
+            path = if parent_url_parsed.path.count('/') > 1
+              parent_url_parsed.path.gsub(parent_url_parsed.path[parent_url_parsed.path.rindex('/')..-1], '')
+            else
+              ''
+            end
+
+            href = if (href[0] == '/' && parent_url_parsed.host[-1] != '/') || (href[0] != '/' && parent_url_parsed.host[-1] == '/')
+              "#{parent_url_parsed.scheme}://#{parent_url_parsed.host}#{path}#{href}"
             else
-              href
-            end             
+              "#{parent_url_parsed.scheme}://#{parent_url_parsed.host}#{path}/#{href}"
+            end
+          else
+            href
+          end             
+        end
+        local_links.uniq
+      end
+
+      def fetch_links_from(url, max_depth = 1, current_depth = 1)
+        local_links = []
+
+        # get all links to benchmark as user behavior
+        response = begin
+          Net::HTTP.get_response(url)
+        rescue OpenSSL::SSL::SSLError => e
+          puts("The url you provided is wrong, please check is it really ssl encrypted")
+          exit
+        rescue Errno::ECONNREFUSED => e
+          puts("Connection error, please check your internet connection or make sure the server is running (it's local)")
+          exit
+        rescue Timeout::Error => e
+          puts("Timeout error: please check the site you're benchmarking")
+          exit          
+        rescue SocketError => e
+          puts e.message
+          exit
+        end
+
+        response = case response
+          when Net::HTTPSuccess
+            response
+          when Net::HTTPRedirection
+            print "redirected to #{response['location']}\n"
+            # we must correct the url, so we could select right inner links then
+            # consider this: you request www.example.com and it redirects you to example.com
+            # but the @url.host will be still www.example.com, so href.match(Regexp.escape(@url.host)) later will fail
+            self.url = response['location']
+            Net::HTTP.get_response(url)
+          when Net::HTTPNotFound
+            return []
+        end
+
+        doc = ::Nokogiri::HTML(response.body)
+        links = select_local_links_from extract_links_from(doc), url
+        local_links += links
+
+        if max_depth > current_depth
+          links.each do |link|
+            local_links << fetch_links_from(URI.parse(link), max_depth, (current_depth + 1))
           end
-          local_links.uniq!
+        end
+        
+        puts "current_depth - #{current_depth}, local_links - #{local_links.inspect}" if @verbose
 
+        local_links.reject{|l| l.empty?}.flatten.uniq
+      end
+
+      def prepare_queue
+        @requests_queue = if @follow_links
+          local_links = fetch_links_from(@url, @follow_links_depth)
           print "Found #{local_links.size} local links: #{local_links.inspect}\n"
           NethttpAb::RequestsQueue.new(local_links)
         else

data/nethttp_ab.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{nethttp_ab}
-  s.version = "0.0.5"
+  s.version = "0.0.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Dmitrii Samoilov"]
-  s.date = %q{2011-04-15}
+  s.date = %q{2011-05-09}
   s.default_executable = %q{nethttp_ab}
   s.description = %q{Simple tool to test and benchmark sites}
   s.email = %q{germaninthetown@gmail.com}
   s.executables = ["nethttp_ab"]
   s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb"]
-  s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb", "nethttp_ab.gemspec", "test/nethttp_ab_test.rb", "test/requests_queue_test.rb", "test/resources/index.html", "test/resources/links.html", "test/resources/links1.html", "test/resources/links2.html", "test/resources/links3.html", "test/resources/links4.html", "test/resources/links5.html", "test/simple_requests_queue_test.rb", "test/url_regexp_test.rb"]
+  s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb", "nethttp_ab.gemspec", "test/nethttp_ab_test.rb", "test/requests_queue_test.rb", "test/resources/depth2/links1.html", "test/resources/depth2/links2.html", "test/resources/depth2/links3.html", "test/resources/depth2/links_11.html", "test/resources/depth2/links_12.html", "test/resources/depth2/links_21.html", "test/resources/depth2/links_31.html", "test/resources/depth2/links_32.html", "test/resources/depth2/links_33.html", "test/resources/index.html", "test/resources/links.html", "test/resources/links1.html", "test/resources/links2.html", "test/resources/links3.html", "test/resources/links4.html", "test/resources/links5.html", "test/resources/links_depth_2.html", "test/simple_requests_queue_test.rb", "test/url_regexp_test.rb"]
   s.homepage = %q{http://github.com/german/nethttp_ab}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Nethttp_ab", "--main", "README"]
   s.require_paths = ["lib"]

data/test/nethttp_ab_test.rb

@@ -5,9 +5,6 @@ require File.dirname(File.expand_path(__FILE__)) + '/../lib/requester.rb'
 
 class NetHttpAbTest < Test::Unit::TestCase
 
-  class TestResponse < Struct.new(:head, :body, :response_code)
-  end
-
   class MySocketStub
     def initialize(body)
       @body = body
@@ -27,7 +24,7 @@ class NetHttpAbTest < Test::Unit::TestCase
 
     Net::HTTP.instance_eval do
       def self.get_response(url)
-        body = File.read(File.join(File.dirname(File.expand_path(__FILE__)), 'resources', url.path))     
+        body = File.read(File.join(File.dirname(File.expand_path(__FILE__)), 'resources', url.path))
         resp = Net::HTTPOK.new(Net::HTTP.version_1_2, '200', '')
         resp.reading_body(MySocketStub.new(body), true){}
         resp.body = body
@@ -58,4 +55,17 @@ class NetHttpAbTest < Test::Unit::TestCase
     assert_equal 5, @requester.successfull_requests
     assert_equal 0, @requester.failed_requests
   end
+
+  def test_follow_links_depth_2
+    @requester.url = "http://localhost/links_depth_2.html"
+    @requester.concurrent_users = 3
+    @requester.follow_links = true
+    @requester.follow_links_depth = 2
+
+    @requester.proceed
+
+    assert_equal 9, @requester.successfull_requests
+    assert_equal 0, @requester.failed_requests
+  end
+
 end

data/test/resources/depth2/links1.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+    <title>Depth 1 / Links #1</title>
+  </head>
+  <body>
+    <h3>Depth 1 / Links #1</h3>
+    <ul>
+      <li><a href="links_11.html">Depth 2 / Links #11</a></li>
+      <li><a href="links_12.html">Depth 2 / Links #12</a></li>
+    </ul>
+  </body>
+</html>

data/test/resources/depth2/links2.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+    <title>Depth 1 / Links #2</title>
+  </head>
+  <body>
+    <h3>Depth 1 / Links #2</h3>
+    <ul>
+      <li><a href="links_21.html">Depth 2 / Links #21</a></li>
+    </ul>
+  </body>
+</html>

data/test/resources/depth2/links3.html

@@ -0,0 +1,13 @@
+<html>
+  <head>
+    <title>Depth 1 / Links #3</title>
+  </head>
+  <body>
+    <h3>Depth 1 / Links #3</h3>
+    <ul>
+      <li><a href="links_31.html">Depth 2 / Links #31</a></li>
+      <li><a href="links_32.html">Depth 2 / Links #32</a></li>
+      <li><a href="links_33.html">Depth 2 / Links #33</a></li>
+    </ul>
+  </body>
+</html>

data/test/resources/depth2/links_11.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #11</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #11</h3>
+  </body>
+</html>

data/test/resources/depth2/links_12.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #12</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #12</h3>
+  </body>
+</html>

data/test/resources/depth2/links_21.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #21</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #21</h3>
+    <a href="links_211.html">Depth 3 / Links #211</a>
+  </body>
+</html>

data/test/resources/depth2/links_31.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #31</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #31</h3>
+  </body>
+</html>

data/test/resources/depth2/links_32.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #32</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #32</h3>
+  </body>
+</html>

data/test/resources/depth2/links_33.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Depth 2 / Links #33</title>
+  </head>
+  <body>
+    <h3>Depth 2 / Links #33</h3>
+  </body>
+</html>

data/test/resources/links_depth_2.html

@@ -0,0 +1,13 @@
+<html>
+  <head>
+    <title>Follow links (depth 2) test</title>
+  </head>
+  <body>
+    <h3>Follow links (depth 2) test</h3>
+    <ul>
+      <li><a href="depth2/links1.html">Link #1</a></li>
+      <li><a href="depth2/links2.html">Link #2</a></li>
+      <li><a href="depth2/links3.html">Link #3</a></li>
+    </ul>
+  </body>
+</html>

data/test/url_regexp_test.rb

@@ -4,7 +4,7 @@ require File.dirname(File.expand_path(__FILE__)) + '/../lib/requester.rb'
 
 class UrlRegexpTest < Test::Unit::TestCase
   def setup
-    @fail_urls = %w{www./yandex.ru yandex.123ru germaninthetown@gmail.com abcde}
+    @fail_urls = %w{www./yandex.ru yandex.123ru germaninthetown@gmail.com abcde index.html}
 
     @correct_urls = %w{www.google.com google.com http://google.com http://mail.ya.ru mail.ya.ru/ mail.ya.ru/test http://www.my-site.org/articles/2011/04/12/123-nethttpab-is-great}
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nethttp_ab
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-04-15 00:00:00.000000000 +03:00
+date: 2011-05-09 00:00:00.000000000 +03:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &75423650 !ruby/object:Gem::Requirement
+  requirement: &82350330 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,7 +22,7 @@ dependencies:
         version: 1.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *75423650
+  version_requirements: *82350330
 description: Simple tool to test and benchmark sites
 email: germaninthetown@gmail.com
 executables:
@@ -50,6 +50,15 @@ files:
 - nethttp_ab.gemspec
 - test/nethttp_ab_test.rb
 - test/requests_queue_test.rb
+- test/resources/depth2/links1.html
+- test/resources/depth2/links2.html
+- test/resources/depth2/links3.html
+- test/resources/depth2/links_11.html
+- test/resources/depth2/links_12.html
+- test/resources/depth2/links_21.html
+- test/resources/depth2/links_31.html
+- test/resources/depth2/links_32.html
+- test/resources/depth2/links_33.html
 - test/resources/index.html
 - test/resources/links.html
 - test/resources/links1.html
@@ -57,6 +66,7 @@ files:
 - test/resources/links3.html
 - test/resources/links4.html
 - test/resources/links5.html
+- test/resources/links_depth_2.html
 - test/simple_requests_queue_test.rb
 - test/url_regexp_test.rb
 has_rdoc: true