nethttp_ab 0.0.4 → 0.0.5

data/CHANGELOG

@@ -1,7 +1,24 @@
-v0.0.4 added correct dependencies to the nethttp_ab.gemspec file [12-04-2011]
+v0.0.5 [15-04-2011]
+* fixed issue #1 ("HTTP request path is empty")
+* following the redirect (if any) on the first request (using Net::HTTP.get_response instead of Net::HTTP::Get) so deleted net.rb file
+* casting all local_links to the standart URL view after first request if option --follow-links
+* removed mocha dependency and rewritten all test suites from nethttp_ab_test.rb with Net::HTTP.get_response overridden
+* changed NethttpAb::Requester::URL_REGEXP (added a white-list of available TLDs)
 
-v0.0.3 fixed a bug in url regexp and added a test suite to cover it, also added a test suite for --follow-links option, changed the way options are parsed - now it's possible to specify both "-n100 -c20" and "-n  100 -c 25" [12-04-2011]
+v0.0.4 [12-04-2011]
+* added correct dependencies to the nethttp_ab.gemspec file 
 
-v0.0.2 deleted trollop gem as a requirement, added nokogiri dependency to gemspec, added MIT license file, fixed small issue with benchmarked site's url, added wrong site address handling  [06-04-2011]
+v0.0.3 [12-04-2011]
+* fixed a bug in url regexp and added a test suite to cover it
+* also added a test suite for --follow-links option
+* changed the way options are parsed - now it's possible to specify both "-n100 -c20" and "-n  100 -c 25"
 
-v0.0.1. first release [05-04-2011]
+v0.0.2 [06-04-2011]
+* deleted trollop gem as a requirement
+* added nokogiri dependency to gemspec
+* added MIT license file
+* fixed small issue with benchmarked site's url
+* added wrong site address handling
+
+v0.0.1. [05-04-2011]
+* first release, w00t!

data/Manifest

@@ -5,7 +5,6 @@ Manifest
 README
 Rakefile
 bin/nethttp_ab
-lib/net.rb
 lib/requester.rb
 lib/requests_queue.rb
 lib/simple_requests_queue.rb

data/Rakefile

@@ -2,10 +2,10 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('nethttp_ab', '0.0.4') do |p|
+Echoe.new('nethttp_ab', '0.0.5') do |p|
   p.description    = "Simple tool to test and benchmark sites"
   p.url            = "http://github.com/german/nethttp_ab"
   p.author         = "Dmitrii Samoilov"
   p.email          = "germaninthetown@gmail.com"
-  p.dependencies   = ["nokogiri >=1.4.2", "mocha"]
+  p.dependencies   = ["nokogiri >=1.4.2"]
 end

data/lib/requester.rb

@@ -1,18 +1,21 @@
 require 'benchmark'
 require 'thread'
+require 'net/http'
 
 require File.dirname(File.expand_path(__FILE__)) + '/requests_queue'
 require File.dirname(File.expand_path(__FILE__)) + '/simple_requests_queue'
-require File.dirname(File.expand_path(__FILE__)) + '/net'
 
 module NethttpAb
   class Requester
-    include NethttpAb::Utility
 
     attr_accessor :successfull_requests
     attr_accessor :failed_requests
 
-    URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,4}/
+    #URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,4}/
+
+    # list of all available TLD domains
+    # http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+    URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+(AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|YE|YT|ZA|ZM|ZW)(\?|\Z|\/)(.*)?/i
 
     def initialize
       @response_length = 0
@@ -44,6 +47,7 @@ module NethttpAb
     end
 
     def print_stats
+      puts
       print "Failed requests: #{@failed_requests}\n"
       print "Succeeded requests: #{@successfull_requests}\n\n"
 
@@ -67,27 +71,58 @@ module NethttpAb
       def prepare_queue
         @requests_queue = if @follow_links
           # get all links to benchmark as user behavior
-          begin
-            http_opened_session = get_http_session(@url)
+          response = begin
+            Net::HTTP.get_response(@url)
           rescue OpenSSL::SSL::SSLError => e
             puts("The url you provided is wrong, please check is it really ssl encrypted")
             exit
           rescue Errno::ECONNREFUSED => e
             puts("Connection error, please check your internet connection or make sure the server is running (it's local)")
             exit
+          rescue Timeout::Error => e
+            puts("Timeout error: please check the site you're benchmarking")
+            exit          
           rescue SocketError => e
             puts e.message
             exit
           end
-
-          req = Net::HTTP::Get.new(@url.path)
-          response = http_opened_session.request(req)
+          
+          response = case response
+            when Net::HTTPSuccess
+              response
+            when Net::HTTPRedirection
+              print "redirected to #{response['location']}\n"
+              # we must correct the url, so we could select right inner links then
+              # consider this: you request www.example.com and it redirects you to example.com
+              # but the @url.host will be still www.example.com, so href.match(Regexp.escape(@url.host)) later will fail
+              self.url = response['location']
+              Net::HTTP.get_response(@url)
+          end
+          
           doc = Nokogiri::HTML(response.body)
-          #puts doc.css('a').map{|el| el.attr('href')}.inspect
-          local_links = doc.css('a').reject{|el| el.attr('rel') == 'nofollow'}.select{|el| el.attr('href').match(Regexp.escape(@url.host)) || (el.attr('href') !~ /^(http|www|javascript)/) }
-          local_links.map!{|el| el.attr('href')}
+
+          local_links = doc.css('a').reject{|el| el.attr('rel') == 'nofollow' || el.attr('href') =~ /^javascript/ || el.attr('onclick')}
+
+          local_links.map!{|el| el.attr('href') }
+
+          local_links = local_links.select{|href| href.match(Regexp.escape(@url.host)) || href !~ URL_REGEXP }
+
+          # we assume that local_links now contains only links inner to tested site
+          local_links.map! do |href|
+            # we must construct a proper url which then could be parsed by URI.parse
+            if !href.match(Regexp.escape(@url.host))
+              href = if (href[0] == '/' && @url.host[-1] != '/') || (href[0] != '/' && @url.host[-1] == '/')
+                "#{@url.scheme}://#{@url.host}#{href}"
+              else
+                "#{@url.scheme}://#{@url.host}/#{href}"
+              end
+            else
+              href
+            end             
+          end
           local_links.uniq!
-          print "Found #{local_links.inspect} local links\n"
+
+          print "Found #{local_links.size} local links: #{local_links.inspect}\n"
           NethttpAb::RequestsQueue.new(local_links)
         else
           NethttpAb::SimpleRequestsQueue.new(@num_of_requests)         
@@ -96,18 +131,18 @@ module NethttpAb
 
       def start_threads
         @concurrent_users.times do
-          begin
-            http_opened_session = get_http_session(@url)
-          rescue OpenSSL::SSL::SSLError => e
-            puts "The url you provided is wrong, please check is it really ssl encrypted"
-            exit
-          rescue Errno::ECONNREFUSED => e
-            puts "Connection error, please check your internet connection or make sure the server is responding"
-            exit
-          rescue SocketError => e
-            puts e.message
-            exit
-          end
+          #begin
+          #  http_opened_session = get_http_session(@url)
+          #rescue OpenSSL::SSL::SSLError => e
+          #  puts "The url you provided is wrong, please check is it really ssl encrypted"
+          #  exit
+          #rescue Errno::ECONNREFUSED => e
+          #  puts "Connection error, please check your internet connection or make sure the server is responding"
+          #  exit
+          #rescue SocketError => e
+          #  puts e.message
+          #  exit
+          #end
 
           @threads << Thread.new do
             while !@requests_queue.empty? do
@@ -115,9 +150,10 @@ module NethttpAb
               if next_url = @requests_queue.lock_next_request
                 req = if @follow_links
                   next_url_parsed = URI.parse(next_url)
-                  Net::HTTP::Get.new(next_url_parsed.path)
+                  next_url_parsed.path = '/' if next_url_parsed.path == "" # ensure we requesting main page (if url is like http://google.com)
+                  next_url_parsed
                 else
-                  Net::HTTP::Get.new(@url.path)
+                  @url
                 end
 
                 # TODO
@@ -125,12 +161,20 @@ module NethttpAb
 
                 @total_time += Benchmark.realtime do
                   begin
-                    response = http_opened_session.request(req)
+                    response = Net::HTTP.get_response(req) #http_opened_session.request(req)
+                    response = case response
+                      when Net::HTTPSuccess
+                        response
+                      when Net::HTTPRedirection
+                        print "redirected to #{response['location']}\n"
+                        Net::HTTP.get_response(URI.parse(response['location']))
+                    end
+                    
+                    print '.' # show progress while processing queue
 
                     @mutex.synchronize do
                       @response_length += response.body.length
-                      @successfull_requests += 1
-                      @requests_queue.release_locked_request(next_url)
+                      @successfull_requests += 1                      
                     end 
                   rescue Net::HTTPBadResponse => e
                     print "An error occured: #{e.message}\n"
@@ -141,6 +185,8 @@ module NethttpAb
                   rescue => e
                     print "An error occured: #{e.message}\n"
                     @failed_requests += 1
+                  ensure
+                    @requests_queue.release_locked_request(next_url)
                   end
                 end
               else

data/nethttp_ab.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{nethttp_ab}
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Dmitrii Samoilov"]
-  s.date = %q{2011-04-12}
+  s.date = %q{2011-04-15}
   s.default_executable = %q{nethttp_ab}
   s.description = %q{Simple tool to test and benchmark sites}
   s.email = %q{germaninthetown@gmail.com}
   s.executables = ["nethttp_ab"]
-  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/nethttp_ab", "lib/net.rb", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb"]
-  s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/net.rb", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb", "nethttp_ab.gemspec", "test/nethttp_ab_test.rb", "test/requests_queue_test.rb", "test/resources/index.html", "test/resources/links.html", "test/resources/links1.html", "test/resources/links2.html", "test/resources/links3.html", "test/resources/links4.html", "test/resources/links5.html", "test/simple_requests_queue_test.rb", "test/url_regexp_test.rb"]
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb"]
+  s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb", "nethttp_ab.gemspec", "test/nethttp_ab_test.rb", "test/requests_queue_test.rb", "test/resources/index.html", "test/resources/links.html", "test/resources/links1.html", "test/resources/links2.html", "test/resources/links3.html", "test/resources/links4.html", "test/resources/links5.html", "test/simple_requests_queue_test.rb", "test/url_regexp_test.rb"]
   s.homepage = %q{http://github.com/german/nethttp_ab}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Nethttp_ab", "--main", "README"]
   s.require_paths = ["lib"]
@@ -26,13 +26,10 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.2"])
-      s.add_runtime_dependency(%q<mocha>, [">= 0"])
     else
       s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
-      s.add_dependency(%q<mocha>, [">= 0"])
     end
   else
     s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
-    s.add_dependency(%q<mocha>, [">= 0"])
   end
 end

data/test/nethttp_ab_test.rb

@@ -1,23 +1,39 @@
 require 'test/unit'
 require 'net/http'
-require 'mocha'
 
 require File.dirname(File.expand_path(__FILE__)) + '/../lib/requester.rb'
 
-class TestResponse < Struct.new(:head, :body, :response_code)
-end
+class NetHttpAbTest < Test::Unit::TestCase
 
-class OpenSession
-  def request(url)
-    body = File.read(File.join(File.dirname(File.expand_path(__FILE__)), 'resources', url.path))
-    TestResponse.new("head", body, 200)
+  class TestResponse < Struct.new(:head, :body, :response_code)
+  end
+
+  class MySocketStub
+    def initialize(body)
+      @body = body
+    end
+
+    def closed?
+      false
+    end
+
+    def read_all(from)
+      @body
+    end
   end
-end
 
-class NetHttpAbTest < Test::Unit::TestCase
   def setup
-    Net::HTTP.any_instance.stubs(:start).returns(OpenSession.new)
     @requester = NethttpAb::Requester.new
+
+    Net::HTTP.instance_eval do
+      def self.get_response(url)
+        body = File.read(File.join(File.dirname(File.expand_path(__FILE__)), 'resources', url.path))     
+        resp = Net::HTTPOK.new(Net::HTTP.version_1_2, '200', '')
+        resp.reading_body(MySocketStub.new(body), true){}
+        resp.body = body
+        resp
+      end
+    end
   end
 
   def test_simple

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nethttp_ab
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-04-12 00:00:00.000000000 +03:00
+date: 2011-04-15 00:00:00.000000000 +03:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &72289600 !ruby/object:Gem::Requirement
+  requirement: &75423650 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,18 +22,7 @@ dependencies:
         version: 1.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *72289600
-- !ruby/object:Gem::Dependency
-  name: mocha
-  requirement: &72289390 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *72289390
+  version_requirements: *75423650
 description: Simple tool to test and benchmark sites
 email: germaninthetown@gmail.com
 executables:
@@ -44,7 +33,6 @@ extra_rdoc_files:
 - LICENSE
 - README
 - bin/nethttp_ab
-- lib/net.rb
 - lib/requester.rb
 - lib/requests_queue.rb
 - lib/simple_requests_queue.rb
@@ -56,7 +44,6 @@ files:
 - README
 - Rakefile
 - bin/nethttp_ab
-- lib/net.rb
 - lib/requester.rb
 - lib/requests_queue.rb
 - lib/simple_requests_queue.rb

data/lib/net.rb

@@ -1,21 +0,0 @@
-require 'net/http'
-require 'uri'
-
-module NethttpAb
-  module Utility
-    def get_http_session(url)
-      http = Net::HTTP.new(url.host, url.port)
-      if url.scheme == "https"
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
-
-      http.read_timeout = 10
-      http.open_timeout = 10
-
-      http_opened_session = http.start
-
-      http_opened_session
-    end
-  end
-end