nethttp_ab 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +21 -4
- data/Manifest +0 -1
- data/Rakefile +2 -2
- data/lib/requester.rb +75 -29
- data/nethttp_ab.gemspec +4 -7
- data/test/nethttp_ab_test.rb +26 -10
- metadata +4 -17
- data/lib/net.rb +0 -21
data/CHANGELOG
CHANGED
@@ -1,7 +1,24 @@
|
|
1
|
-
v0.0.
|
1
|
+
v0.0.5 [15-04-2011]
|
2
|
+
* fixed issue #1 ("HTTP request path is empty")
|
3
|
+
* following the redirect (if any) on the first request (using Net::HTTP.get_response instead of Net::HTTP::Get) so deleted net.rb file
|
4
|
+
* casting all local_links to the standart URL view after first request if option --follow-links
|
5
|
+
* removed mocha dependency and rewritten all test suites from nethttp_ab_test.rb with Net::HTTP.get_response overridden
|
6
|
+
* changed NethttpAb::Requester::URL_REGEXP (added a white-list of available TLDs)
|
2
7
|
|
3
|
-
v0.0.
|
8
|
+
v0.0.4 [12-04-2011]
|
9
|
+
* added correct dependencies to the nethttp_ab.gemspec file
|
4
10
|
|
5
|
-
v0.0.
|
11
|
+
v0.0.3 [12-04-2011]
|
12
|
+
* fixed a bug in url regexp and added a test suite to cover it
|
13
|
+
* also added a test suite for --follow-links option
|
14
|
+
* changed the way options are parsed - now it's possible to specify both "-n100 -c20" and "-n 100 -c 25"
|
6
15
|
|
7
|
-
v0.0.
|
16
|
+
v0.0.2 [06-04-2011]
|
17
|
+
* deleted trollop gem as a requirement
|
18
|
+
* added nokogiri dependency to gemspec
|
19
|
+
* added MIT license file
|
20
|
+
* fixed small issue with benchmarked site's url
|
21
|
+
* added wrong site address handling
|
22
|
+
|
23
|
+
v0.0.1. [05-04-2011]
|
24
|
+
* first release, w00t!
|
data/Manifest
CHANGED
data/Rakefile
CHANGED
@@ -2,10 +2,10 @@ require 'rubygems'
|
|
2
2
|
require 'rake'
|
3
3
|
require 'echoe'
|
4
4
|
|
5
|
-
Echoe.new('nethttp_ab', '0.0.
|
5
|
+
Echoe.new('nethttp_ab', '0.0.5') do |p|
|
6
6
|
p.description = "Simple tool to test and benchmark sites"
|
7
7
|
p.url = "http://github.com/german/nethttp_ab"
|
8
8
|
p.author = "Dmitrii Samoilov"
|
9
9
|
p.email = "germaninthetown@gmail.com"
|
10
|
-
p.dependencies = ["nokogiri >=1.4.2"
|
10
|
+
p.dependencies = ["nokogiri >=1.4.2"]
|
11
11
|
end
|
data/lib/requester.rb
CHANGED
@@ -1,18 +1,21 @@
|
|
1
1
|
require 'benchmark'
|
2
2
|
require 'thread'
|
3
|
+
require 'net/http'
|
3
4
|
|
4
5
|
require File.dirname(File.expand_path(__FILE__)) + '/requests_queue'
|
5
6
|
require File.dirname(File.expand_path(__FILE__)) + '/simple_requests_queue'
|
6
|
-
require File.dirname(File.expand_path(__FILE__)) + '/net'
|
7
7
|
|
8
8
|
module NethttpAb
|
9
9
|
class Requester
|
10
|
-
include NethttpAb::Utility
|
11
10
|
|
12
11
|
attr_accessor :successfull_requests
|
13
12
|
attr_accessor :failed_requests
|
14
13
|
|
15
|
-
URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,4}/
|
14
|
+
#URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,4}/
|
15
|
+
|
16
|
+
# list of all available TLD domains
|
17
|
+
# http://data.iana.org/TLD/tlds-alpha-by-domain.txt
|
18
|
+
URL_REGEXP = /^(https?:\/\/)?(?:[a-zA-Z0-9-]+\.)+(AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|YE|YT|ZA|ZM|ZW)(\?|\Z|\/)(.*)?/i
|
16
19
|
|
17
20
|
def initialize
|
18
21
|
@response_length = 0
|
@@ -44,6 +47,7 @@ module NethttpAb
|
|
44
47
|
end
|
45
48
|
|
46
49
|
def print_stats
|
50
|
+
puts
|
47
51
|
print "Failed requests: #{@failed_requests}\n"
|
48
52
|
print "Succeeded requests: #{@successfull_requests}\n\n"
|
49
53
|
|
@@ -67,27 +71,58 @@ module NethttpAb
|
|
67
71
|
def prepare_queue
|
68
72
|
@requests_queue = if @follow_links
|
69
73
|
# get all links to benchmark as user behavior
|
70
|
-
begin
|
71
|
-
|
74
|
+
response = begin
|
75
|
+
Net::HTTP.get_response(@url)
|
72
76
|
rescue OpenSSL::SSL::SSLError => e
|
73
77
|
puts("The url you provided is wrong, please check is it really ssl encrypted")
|
74
78
|
exit
|
75
79
|
rescue Errno::ECONNREFUSED => e
|
76
80
|
puts("Connection error, please check your internet connection or make sure the server is running (it's local)")
|
77
81
|
exit
|
82
|
+
rescue Timeout::Error => e
|
83
|
+
puts("Timeout error: please check the site you're benchmarking")
|
84
|
+
exit
|
78
85
|
rescue SocketError => e
|
79
86
|
puts e.message
|
80
87
|
exit
|
81
88
|
end
|
82
|
-
|
83
|
-
|
84
|
-
|
89
|
+
|
90
|
+
response = case response
|
91
|
+
when Net::HTTPSuccess
|
92
|
+
response
|
93
|
+
when Net::HTTPRedirection
|
94
|
+
print "redirected to #{response['location']}\n"
|
95
|
+
# we must correct the url, so we could select right inner links then
|
96
|
+
# consider this: you request www.example.com and it redirects you to example.com
|
97
|
+
# but the @url.host will be still www.example.com, so href.match(Regexp.escape(@url.host)) later will fail
|
98
|
+
self.url = response['location']
|
99
|
+
Net::HTTP.get_response(@url)
|
100
|
+
end
|
101
|
+
|
85
102
|
doc = Nokogiri::HTML(response.body)
|
86
|
-
|
87
|
-
local_links = doc.css('a').reject{|el| el.attr('rel') == 'nofollow'
|
88
|
-
|
103
|
+
|
104
|
+
local_links = doc.css('a').reject{|el| el.attr('rel') == 'nofollow' || el.attr('href') =~ /^javascript/ || el.attr('onclick')}
|
105
|
+
|
106
|
+
local_links.map!{|el| el.attr('href') }
|
107
|
+
|
108
|
+
local_links = local_links.select{|href| href.match(Regexp.escape(@url.host)) || href !~ URL_REGEXP }
|
109
|
+
|
110
|
+
# we assume that local_links now contains only links inner to tested site
|
111
|
+
local_links.map! do |href|
|
112
|
+
# we must construct a proper url which then could be parsed by URI.parse
|
113
|
+
if !href.match(Regexp.escape(@url.host))
|
114
|
+
href = if (href[0] == '/' && @url.host[-1] != '/') || (href[0] != '/' && @url.host[-1] == '/')
|
115
|
+
"#{@url.scheme}://#{@url.host}#{href}"
|
116
|
+
else
|
117
|
+
"#{@url.scheme}://#{@url.host}/#{href}"
|
118
|
+
end
|
119
|
+
else
|
120
|
+
href
|
121
|
+
end
|
122
|
+
end
|
89
123
|
local_links.uniq!
|
90
|
-
|
124
|
+
|
125
|
+
print "Found #{local_links.size} local links: #{local_links.inspect}\n"
|
91
126
|
NethttpAb::RequestsQueue.new(local_links)
|
92
127
|
else
|
93
128
|
NethttpAb::SimpleRequestsQueue.new(@num_of_requests)
|
@@ -96,18 +131,18 @@ module NethttpAb
|
|
96
131
|
|
97
132
|
def start_threads
|
98
133
|
@concurrent_users.times do
|
99
|
-
begin
|
100
|
-
|
101
|
-
rescue OpenSSL::SSL::SSLError => e
|
102
|
-
|
103
|
-
|
104
|
-
rescue Errno::ECONNREFUSED => e
|
105
|
-
|
106
|
-
|
107
|
-
rescue SocketError => e
|
108
|
-
|
109
|
-
|
110
|
-
end
|
134
|
+
#begin
|
135
|
+
# http_opened_session = get_http_session(@url)
|
136
|
+
#rescue OpenSSL::SSL::SSLError => e
|
137
|
+
# puts "The url you provided is wrong, please check is it really ssl encrypted"
|
138
|
+
# exit
|
139
|
+
#rescue Errno::ECONNREFUSED => e
|
140
|
+
# puts "Connection error, please check your internet connection or make sure the server is responding"
|
141
|
+
# exit
|
142
|
+
#rescue SocketError => e
|
143
|
+
# puts e.message
|
144
|
+
# exit
|
145
|
+
#end
|
111
146
|
|
112
147
|
@threads << Thread.new do
|
113
148
|
while !@requests_queue.empty? do
|
@@ -115,9 +150,10 @@ module NethttpAb
|
|
115
150
|
if next_url = @requests_queue.lock_next_request
|
116
151
|
req = if @follow_links
|
117
152
|
next_url_parsed = URI.parse(next_url)
|
118
|
-
|
153
|
+
next_url_parsed.path = '/' if next_url_parsed.path == "" # ensure we requesting main page (if url is like http://google.com)
|
154
|
+
next_url_parsed
|
119
155
|
else
|
120
|
-
|
156
|
+
@url
|
121
157
|
end
|
122
158
|
|
123
159
|
# TODO
|
@@ -125,12 +161,20 @@ module NethttpAb
|
|
125
161
|
|
126
162
|
@total_time += Benchmark.realtime do
|
127
163
|
begin
|
128
|
-
response = http_opened_session.request(req)
|
164
|
+
response = Net::HTTP.get_response(req) #http_opened_session.request(req)
|
165
|
+
response = case response
|
166
|
+
when Net::HTTPSuccess
|
167
|
+
response
|
168
|
+
when Net::HTTPRedirection
|
169
|
+
print "redirected to #{response['location']}\n"
|
170
|
+
Net::HTTP.get_response(URI.parse(response['location']))
|
171
|
+
end
|
172
|
+
|
173
|
+
print '.' # show progress while processing queue
|
129
174
|
|
130
175
|
@mutex.synchronize do
|
131
176
|
@response_length += response.body.length
|
132
|
-
@successfull_requests += 1
|
133
|
-
@requests_queue.release_locked_request(next_url)
|
177
|
+
@successfull_requests += 1
|
134
178
|
end
|
135
179
|
rescue Net::HTTPBadResponse => e
|
136
180
|
print "An error occured: #{e.message}\n"
|
@@ -141,6 +185,8 @@ module NethttpAb
|
|
141
185
|
rescue => e
|
142
186
|
print "An error occured: #{e.message}\n"
|
143
187
|
@failed_requests += 1
|
188
|
+
ensure
|
189
|
+
@requests_queue.release_locked_request(next_url)
|
144
190
|
end
|
145
191
|
end
|
146
192
|
else
|
data/nethttp_ab.gemspec
CHANGED
@@ -2,17 +2,17 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{nethttp_ab}
|
5
|
-
s.version = "0.0.
|
5
|
+
s.version = "0.0.5"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Dmitrii Samoilov"]
|
9
|
-
s.date = %q{2011-04-
|
9
|
+
s.date = %q{2011-04-15}
|
10
10
|
s.default_executable = %q{nethttp_ab}
|
11
11
|
s.description = %q{Simple tool to test and benchmark sites}
|
12
12
|
s.email = %q{germaninthetown@gmail.com}
|
13
13
|
s.executables = ["nethttp_ab"]
|
14
|
-
s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/nethttp_ab", "lib/
|
15
|
-
s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/
|
14
|
+
s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb"]
|
15
|
+
s.files = ["CHANGELOG", "Gemfile", "LICENSE", "Manifest", "README", "Rakefile", "bin/nethttp_ab", "lib/requester.rb", "lib/requests_queue.rb", "lib/simple_requests_queue.rb", "nethttp_ab.gemspec", "test/nethttp_ab_test.rb", "test/requests_queue_test.rb", "test/resources/index.html", "test/resources/links.html", "test/resources/links1.html", "test/resources/links2.html", "test/resources/links3.html", "test/resources/links4.html", "test/resources/links5.html", "test/simple_requests_queue_test.rb", "test/url_regexp_test.rb"]
|
16
16
|
s.homepage = %q{http://github.com/german/nethttp_ab}
|
17
17
|
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Nethttp_ab", "--main", "README"]
|
18
18
|
s.require_paths = ["lib"]
|
@@ -26,13 +26,10 @@ Gem::Specification.new do |s|
|
|
26
26
|
|
27
27
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
28
28
|
s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.2"])
|
29
|
-
s.add_runtime_dependency(%q<mocha>, [">= 0"])
|
30
29
|
else
|
31
30
|
s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
|
32
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
33
31
|
end
|
34
32
|
else
|
35
33
|
s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
|
36
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
37
34
|
end
|
38
35
|
end
|
data/test/nethttp_ab_test.rb
CHANGED
@@ -1,23 +1,39 @@
|
|
1
1
|
require 'test/unit'
|
2
2
|
require 'net/http'
|
3
|
-
require 'mocha'
|
4
3
|
|
5
4
|
require File.dirname(File.expand_path(__FILE__)) + '/../lib/requester.rb'
|
6
5
|
|
7
|
-
class
|
8
|
-
end
|
6
|
+
class NetHttpAbTest < Test::Unit::TestCase
|
9
7
|
|
10
|
-
class
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
class TestResponse < Struct.new(:head, :body, :response_code)
|
9
|
+
end
|
10
|
+
|
11
|
+
class MySocketStub
|
12
|
+
def initialize(body)
|
13
|
+
@body = body
|
14
|
+
end
|
15
|
+
|
16
|
+
def closed?
|
17
|
+
false
|
18
|
+
end
|
19
|
+
|
20
|
+
def read_all(from)
|
21
|
+
@body
|
22
|
+
end
|
14
23
|
end
|
15
|
-
end
|
16
24
|
|
17
|
-
class NetHttpAbTest < Test::Unit::TestCase
|
18
25
|
def setup
|
19
|
-
Net::HTTP.any_instance.stubs(:start).returns(OpenSession.new)
|
20
26
|
@requester = NethttpAb::Requester.new
|
27
|
+
|
28
|
+
Net::HTTP.instance_eval do
|
29
|
+
def self.get_response(url)
|
30
|
+
body = File.read(File.join(File.dirname(File.expand_path(__FILE__)), 'resources', url.path))
|
31
|
+
resp = Net::HTTPOK.new(Net::HTTP.version_1_2, '200', '')
|
32
|
+
resp.reading_body(MySocketStub.new(body), true){}
|
33
|
+
resp.body = body
|
34
|
+
resp
|
35
|
+
end
|
36
|
+
end
|
21
37
|
end
|
22
38
|
|
23
39
|
def test_simple
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nethttp_ab
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,12 +9,12 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-04-
|
12
|
+
date: 2011-04-15 00:00:00.000000000 +03:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: nokogiri
|
17
|
-
requirement: &
|
17
|
+
requirement: &75423650 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
@@ -22,18 +22,7 @@ dependencies:
|
|
22
22
|
version: 1.4.2
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
26
|
-
- !ruby/object:Gem::Dependency
|
27
|
-
name: mocha
|
28
|
-
requirement: &72289390 !ruby/object:Gem::Requirement
|
29
|
-
none: false
|
30
|
-
requirements:
|
31
|
-
- - ! '>='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: *72289390
|
25
|
+
version_requirements: *75423650
|
37
26
|
description: Simple tool to test and benchmark sites
|
38
27
|
email: germaninthetown@gmail.com
|
39
28
|
executables:
|
@@ -44,7 +33,6 @@ extra_rdoc_files:
|
|
44
33
|
- LICENSE
|
45
34
|
- README
|
46
35
|
- bin/nethttp_ab
|
47
|
-
- lib/net.rb
|
48
36
|
- lib/requester.rb
|
49
37
|
- lib/requests_queue.rb
|
50
38
|
- lib/simple_requests_queue.rb
|
@@ -56,7 +44,6 @@ files:
|
|
56
44
|
- README
|
57
45
|
- Rakefile
|
58
46
|
- bin/nethttp_ab
|
59
|
-
- lib/net.rb
|
60
47
|
- lib/requester.rb
|
61
48
|
- lib/requests_queue.rb
|
62
49
|
- lib/simple_requests_queue.rb
|
data/lib/net.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'net/http'
|
2
|
-
require 'uri'
|
3
|
-
|
4
|
-
module NethttpAb
|
5
|
-
module Utility
|
6
|
-
def get_http_session(url)
|
7
|
-
http = Net::HTTP.new(url.host, url.port)
|
8
|
-
if url.scheme == "https"
|
9
|
-
http.use_ssl = true
|
10
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
11
|
-
end
|
12
|
-
|
13
|
-
http.read_timeout = 10
|
14
|
-
http.open_timeout = 10
|
15
|
-
|
16
|
-
http_opened_session = http.start
|
17
|
-
|
18
|
-
http_opened_session
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|