http_status_checker 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{url_alive.gemspec → http_status_checker.gemspec} +2 -1
- data/lib/http_status_checker/base.rb +10 -7
- data/lib/http_status_checker/version.rb +1 -1
- data/spec/http_status_checker/base_spec.rb +56 -48
- data/spec/http_status_checker/connection_spec.rb +28 -12
- data/spec/spec_helper.rb +8 -13
- data/spec/vcr/base/atnd_http_url.yml +109 -0
- data/spec/vcr/base/morizyun_css_morizyun_js.yml +1938 -0
- data/spec/vcr/base/redirect_url.yml +522 -0
- data/spec/vcr/base/valid_url.yml +441 -0
- data/spec/vcr/connection/atnd_https_url.yml +56 -0
- data/spec/vcr/connection/doorkeeper_url.yml +967 -0
- data/spec/vcr/connection/redirect_url.yml +84 -0
- data/spec/vcr/connection/valid_url.yml +441 -0
- metadata +34 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fd51837fa68f9996098bd0c4aae7163ee9292408
|
4
|
+
data.tar.gz: 517c6dc2edb3ff9b4c79e5e9203b1bb1ea46f375
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86f1f451dcddf5104c6c131f5407f81517a9ed24681d2ea78836d65c4dada52923e8651872acb3a41ab8e9b80ca912d7157d1ae86311e4f87f57b75c77dd3865
|
7
|
+
data.tar.gz: 86a9bfb5515f263c9a50b20f24c1b1bd6bedf0526e4e7f9f23cbab1aa6f6cb3fa38a08d569996cec5202cfc16ae1e65b45c68b754fc6007d079ae7ec0a00a2c3
|
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_dependency 'parallel'
|
24
24
|
spec.add_development_dependency 'bundler', '~> 1.7'
|
25
25
|
spec.add_development_dependency 'rake'
|
26
|
-
spec.add_development_dependency 'fakeweb'
|
27
26
|
spec.add_development_dependency 'rspec'
|
27
|
+
spec.add_development_dependency 'webmock'
|
28
|
+
spec.add_development_dependency 'vcr'
|
28
29
|
end
|
@@ -2,7 +2,7 @@ module HttpStatusChecker
|
|
2
2
|
module Base
|
3
3
|
THREAD_LIMIT = 5.freeze
|
4
4
|
REDIRECT_MAX = 5.freeze
|
5
|
-
RETRY_MAX =
|
5
|
+
RETRY_MAX = 2.freeze
|
6
6
|
|
7
7
|
def check(urls, wait_sec = 1)
|
8
8
|
results = []
|
@@ -10,7 +10,7 @@ module HttpStatusChecker
|
|
10
10
|
host_hash = to_host_hash(urls)
|
11
11
|
Parallel.each(host_hash, in_threads: host_hash.keys.count) do |_, urls|
|
12
12
|
urls.map.with_index(1) do |url, idx|
|
13
|
-
results << get_response(url)
|
13
|
+
results << get_response(url, wait_sec)
|
14
14
|
sleep(wait_sec) if urls.count != idx
|
15
15
|
end
|
16
16
|
end
|
@@ -20,19 +20,22 @@ module HttpStatusChecker
|
|
20
20
|
|
21
21
|
private
|
22
22
|
|
23
|
-
def get_response(url, redirect_url = nil, redirect_count = 0, retry_count = 0)
|
23
|
+
def get_response(url, wait_sec, redirect_url = nil, redirect_count = 0, retry_count = 0)
|
24
24
|
result = HttpStatusChecker::Connection.get_header(redirect_url || url)
|
25
|
-
parse_response(redirect_count, redirect_url, result, url)
|
25
|
+
parse_response(redirect_count, redirect_url, result, url, wait_sec)
|
26
26
|
rescue => e
|
27
|
-
|
27
|
+
if retry_count < RETRY_MAX
|
28
|
+
sleep (wait_sec)
|
29
|
+
get_response(url, wait_sec, nil, redirect_count + 1, retry_count + 1)
|
30
|
+
end
|
28
31
|
{ url => { code: result ? result.code : nil, is_alive: false, error: e.message } }
|
29
32
|
end
|
30
33
|
|
31
|
-
def parse_response(redirect_count, redirect_url, result, url)
|
34
|
+
def parse_response(redirect_count, redirect_url, result, url, wait_sec)
|
32
35
|
location_url = result['location']
|
33
36
|
if !location_url.nil? && (redirect_url || url) != location_url
|
34
37
|
raise InvalidRedirectError if redirect_count > REDIRECT_MAX
|
35
|
-
get_response(url, location_url, redirect_count + 1, 0)
|
38
|
+
get_response(url, wait_sec, location_url, redirect_count + 1, 0)
|
36
39
|
elsif result.code =~ /^(2|3)[0-9]{2}$/
|
37
40
|
{ url => { code: result.code, is_alive: true, redirect_url: redirect_url } }
|
38
41
|
else
|
@@ -6,84 +6,98 @@ describe HttpStatusChecker::Connection do
|
|
6
6
|
|
7
7
|
describe '.check' do
|
8
8
|
context 'when set valid url' do
|
9
|
+
before do
|
10
|
+
VCR.use_cassette 'base/valid_url' do
|
11
|
+
@response = HttpStatusChecker.check(valid_url)
|
12
|
+
end
|
13
|
+
end
|
9
14
|
it 'returns is_alive = true, redirect = nil, error = nil' do
|
10
|
-
response
|
11
|
-
expect(response.first[valid_url][:
|
12
|
-
expect(response.first[valid_url][:
|
13
|
-
expect(response.first[valid_url][:error]).to be_nil
|
15
|
+
expect(@response.first[valid_url][:is_alive]).to eq(true)
|
16
|
+
expect(@response.first[valid_url][:redirect_url]).to be_nil
|
17
|
+
expect(@response.first[valid_url][:error]).to be_nil
|
14
18
|
end
|
15
19
|
end
|
16
20
|
|
17
21
|
context 'when set valid url with invalid http status' do
|
18
22
|
let!(:atnd_http_url) { 'http://atnd.org/events/14386' }
|
19
23
|
let!(:atnd_https_url) { 'https://atnd.org/events/14386' }
|
24
|
+
before do
|
25
|
+
VCR.use_cassette 'base/atnd_http_url' do
|
26
|
+
@response = HttpStatusChecker.check(atnd_http_url)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
20
30
|
it 'returns is_alive = true, redirect = nil, error = nil' do
|
21
|
-
response
|
22
|
-
expect(response.first[atnd_http_url][:
|
23
|
-
expect(response.first[atnd_http_url][:
|
24
|
-
expect(response.first[atnd_http_url][:error]).to be_nil
|
31
|
+
expect(@response.first[atnd_http_url][:is_alive]).to eq(true)
|
32
|
+
expect(@response.first[atnd_http_url][:redirect_url]).to eq(atnd_https_url)
|
33
|
+
expect(@response.first[atnd_http_url][:error]).to be_nil
|
25
34
|
end
|
26
35
|
end
|
27
36
|
|
28
37
|
context 'when set 2 urls' do
|
29
38
|
let!(:morizyun_css) { 'http://morizyun.github.io/blog/css3-html-front-coding-book-review/' }
|
30
39
|
let!(:morizyun_js) { 'http://morizyun.github.io/blog/javascript-learning-tech-yourself_01/' }
|
31
|
-
|
32
|
-
|
33
|
-
|
40
|
+
before do
|
41
|
+
VCR.use_cassette 'base/morizyun_css_morizyun_js' do
|
42
|
+
@time = Benchmark.realtime do
|
43
|
+
@response = HttpStatusChecker.check([morizyun_css, morizyun_js])
|
44
|
+
end
|
34
45
|
end
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
expect(parse(response,
|
39
|
-
expect(
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'returns is_alive = true, error = nil' do
|
49
|
+
expect(parse(@response, morizyun_css)[:is_alive]).to eq(true)
|
50
|
+
expect(parse(@response, morizyun_css)[:error]).to be_nil
|
51
|
+
expect(parse(@response, morizyun_js)[:is_alive]).to eq(true)
|
52
|
+
expect(parse(@response, morizyun_js)[:error]).to be_nil
|
53
|
+
expect(@time).to be >= 1.0
|
40
54
|
end
|
41
55
|
end
|
42
56
|
|
43
57
|
context 'when set 2 urls & wait_sec = 2' do
|
44
58
|
let!(:morizyun_css) { 'http://morizyun.github.io/blog/css3-html-front-coding-book-review/' }
|
45
59
|
let!(:morizyun_js) { 'http://morizyun.github.io/blog/javascript-learning-tech-yourself_01/' }
|
46
|
-
|
47
|
-
|
48
|
-
|
60
|
+
before do
|
61
|
+
VCR.use_cassette 'base/morizyun_css_morizyun_js' do
|
62
|
+
@time = Benchmark.realtime do
|
63
|
+
@response = HttpStatusChecker.check([morizyun_css, morizyun_js], wait_sec = 2)
|
64
|
+
end
|
49
65
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
expect(parse(response,
|
54
|
-
expect(
|
66
|
+
end
|
67
|
+
|
68
|
+
it 'returns is_alive = true, error = nil' do
|
69
|
+
expect(parse(@response, morizyun_css)[:is_alive]).to eq(true)
|
70
|
+
expect(parse(@response, morizyun_css)[:error]).to be_nil
|
71
|
+
expect(parse(@response, morizyun_js)[:is_alive]).to eq(true)
|
72
|
+
expect(parse(@response, morizyun_js)[:error]).to be_nil
|
73
|
+
expect(@time).to be >= 2.0
|
55
74
|
end
|
56
75
|
end
|
57
76
|
|
58
77
|
context 'when set http redirect url' do
|
78
|
+
before do
|
79
|
+
VCR.use_cassette 'base/redirect_url' do
|
80
|
+
@response = HttpStatusChecker.check(redirect_url)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
59
84
|
it 'returns is_alive = true, redirect = valid_url, error = nil' do
|
60
|
-
response
|
61
|
-
expect(response.first[redirect_url][:
|
62
|
-
expect(response.first[redirect_url][:
|
63
|
-
expect(response.first[redirect_url][:error]).to be_nil
|
85
|
+
expect(@response.first[redirect_url][:is_alive]).to eq(true)
|
86
|
+
expect(@response.first[redirect_url][:redirect_url]).to eq(valid_url)
|
87
|
+
expect(@response.first[redirect_url][:error]).to be_nil
|
64
88
|
end
|
65
89
|
end
|
66
90
|
|
67
91
|
context 'when set http invalid url' do
|
68
92
|
let!(:invalid_url) { 'http://www.nothing-dummy.com/' }
|
69
93
|
it 'returns is_alive = false, redirect = nil, error present' do
|
70
|
-
response = HttpStatusChecker.check(invalid_url)
|
71
|
-
expect(response.first[invalid_url][:is_alive]).to eq(false)
|
72
|
-
expect(response.first[invalid_url][:redirect_url]).to be_nil
|
73
|
-
expect(response.first[invalid_url][:error]).not_to be_nil
|
94
|
+
@response = HttpStatusChecker.check(invalid_url)
|
95
|
+
expect(@response.first[invalid_url][:is_alive]).to eq(false)
|
96
|
+
expect(@response.first[invalid_url][:redirect_url]).to be_nil
|
97
|
+
expect(@response.first[invalid_url][:error]).not_to be_nil
|
74
98
|
end
|
75
99
|
end
|
76
100
|
|
77
|
-
context 'when set 404 error' do
|
78
|
-
let!(:morizyun_404) { 'http://morizyun.github.io/404/' }
|
79
|
-
it 'returns is_alive = false, redirect = nil, error present' do
|
80
|
-
response = HttpStatusChecker.check(morizyun_404)
|
81
|
-
expect(response.first[morizyun_404][:code]).to eq('404')
|
82
|
-
expect(response.first[morizyun_404][:is_alive]).to eq(false)
|
83
|
-
expect(response.first[morizyun_404][:redirect_url]).to be_nil
|
84
|
-
expect(response.first[morizyun_404][:error]).not_to be_nil
|
85
|
-
end
|
86
|
-
end
|
87
101
|
end
|
88
102
|
|
89
103
|
describe '.to_host_hash' do
|
@@ -102,11 +116,5 @@ describe HttpStatusChecker::Connection do
|
|
102
116
|
end
|
103
117
|
end
|
104
118
|
|
105
|
-
def measure_time(&block)
|
106
|
-
start_at = Time.now
|
107
|
-
result = block.call
|
108
|
-
finish_at = Time.now
|
109
|
-
return [result, finish_at - start_at]
|
110
|
-
end
|
111
119
|
end
|
112
120
|
|
@@ -9,34 +9,50 @@ describe HttpStatusChecker::Connection do
|
|
9
9
|
let!(:doorkeeper_url) { 'http://mashupawards.doorkeeper.jp/events/18590' }
|
10
10
|
|
11
11
|
context 'when get http valid url' do
|
12
|
+
before do
|
13
|
+
VCR.use_cassette 'connection/valid_url' do
|
14
|
+
@response = HttpStatusChecker::Connection.get_header(valid_url)
|
15
|
+
end
|
16
|
+
end
|
12
17
|
it 'return Net::HTTPOK response' do
|
13
|
-
response
|
14
|
-
expect(response
|
15
|
-
expect(response['location']).to be_nil
|
18
|
+
expect(@response.is_a?(Net::HTTPOK)).to be == true
|
19
|
+
expect(@response['location']).to be_nil
|
16
20
|
end
|
17
21
|
end
|
18
22
|
|
19
23
|
context 'when get http redirect url' do
|
24
|
+
before do
|
25
|
+
VCR.use_cassette 'connection/redirect_url' do
|
26
|
+
@response = HttpStatusChecker::Connection.get_header(redirect_url)
|
27
|
+
end
|
28
|
+
end
|
20
29
|
it 'return Net::HTTPRedirection response' do
|
21
|
-
response
|
22
|
-
expect(response
|
23
|
-
expect(response['location']).to eq(valid_url)
|
30
|
+
expect(@response.is_a?(Net::HTTPRedirection)).to be == true
|
31
|
+
expect(@response['location']).to eq(valid_url)
|
24
32
|
end
|
25
33
|
end
|
26
34
|
|
27
35
|
context 'when get invalid http status(atnd.org)' do
|
36
|
+
before do
|
37
|
+
VCR.use_cassette 'connection/atnd_https_url' do
|
38
|
+
@response = HttpStatusChecker::Connection.get_header(atnd_https_url)
|
39
|
+
end
|
40
|
+
end
|
28
41
|
it 'return Net::HTTPRedirection response' do
|
29
|
-
response
|
30
|
-
expect(response
|
31
|
-
expect(response['location']).to eq(atnd_https_url)
|
42
|
+
expect(@response.is_a?(Net::HTTPRedirection)).to be == true
|
43
|
+
expect(@response['location']).to eq(atnd_https_url)
|
32
44
|
end
|
33
45
|
end
|
34
46
|
|
35
47
|
context 'when get http url' do
|
48
|
+
before do
|
49
|
+
VCR.use_cassette 'connection/doorkeeper_url' do
|
50
|
+
@response = HttpStatusChecker::Connection.get_header(doorkeeper_url)
|
51
|
+
end
|
52
|
+
end
|
36
53
|
it 'return Net::HTTPRedirection response' do
|
37
|
-
response
|
38
|
-
expect(response
|
39
|
-
expect(response['location']).to be_nil
|
54
|
+
expect(@response.is_a?(Net::HTTPOK)).to be == true
|
55
|
+
expect(@response['location']).to be_nil
|
40
56
|
end
|
41
57
|
end
|
42
58
|
|
data/spec/spec_helper.rb
CHANGED
@@ -13,20 +13,15 @@ end
|
|
13
13
|
$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
14
14
|
require 'http_status_checker'
|
15
15
|
require 'rspec'
|
16
|
-
require '
|
17
|
-
|
18
|
-
def fixture(path)
|
19
|
-
File.read("#{File.dirname(__FILE__)}/fixtures/#{path}")
|
20
|
-
end
|
21
|
-
|
22
|
-
def stub_get(path, fixture_path, options={})
|
23
|
-
opts = {
|
24
|
-
:body => fixture(fixture_path),
|
25
|
-
:content_type => 'application/json; charset=utf-8'
|
26
|
-
}.merge(options)
|
27
|
-
FakeWeb.register_uri(:get, "#{path}", opts)
|
28
|
-
end
|
16
|
+
require 'vcr'
|
17
|
+
require 'benchmark'
|
29
18
|
|
30
19
|
RSpec.configure do |config|
|
31
20
|
config.order = 'random'
|
21
|
+
end
|
22
|
+
|
23
|
+
VCR.configure do |c|
|
24
|
+
c.cassette_library_dir = 'spec/vcr'
|
25
|
+
c.hook_into :webmock
|
26
|
+
c.allow_http_connections_when_no_cassette = true
|
32
27
|
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://atnd.org/events/14386
|
6
|
+
body:
|
7
|
+
encoding: US-ASCII
|
8
|
+
string: ''
|
9
|
+
headers:
|
10
|
+
Accept-Encoding:
|
11
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
12
|
+
Accept:
|
13
|
+
- "*/*"
|
14
|
+
User-Agent:
|
15
|
+
- Ruby
|
16
|
+
response:
|
17
|
+
status:
|
18
|
+
code: 302
|
19
|
+
message: Found
|
20
|
+
headers:
|
21
|
+
Date:
|
22
|
+
- Sat, 27 Dec 2014 10:28:38 GMT
|
23
|
+
Server:
|
24
|
+
- Apache
|
25
|
+
X-Frame-Options:
|
26
|
+
- SAMEORIGIN
|
27
|
+
X-Xss-Protection:
|
28
|
+
- 1; mode=block
|
29
|
+
X-Content-Type-Options:
|
30
|
+
- nosniff
|
31
|
+
Cache-Control:
|
32
|
+
- no-cache
|
33
|
+
X-Request-Id:
|
34
|
+
- 298c28d8-9471-4a19-9e9e-7a3b3a4f338f
|
35
|
+
X-Runtime:
|
36
|
+
- '0.005871'
|
37
|
+
X-Powered-By:
|
38
|
+
- Phusion Passenger 4.0.42
|
39
|
+
Set-Cookie:
|
40
|
+
- request_method=GET; path=/
|
41
|
+
Location:
|
42
|
+
- https://atnd.org/events/14386
|
43
|
+
Status:
|
44
|
+
- 302 Found
|
45
|
+
P3p:
|
46
|
+
- CP="NON DSP COR CURa ADMa DEVa CUSo TAIa PSDo OUR BUS UNI COM NAV STA"
|
47
|
+
Transfer-Encoding:
|
48
|
+
- chunked
|
49
|
+
Content-Type:
|
50
|
+
- text/html; charset=utf-8
|
51
|
+
body:
|
52
|
+
encoding: UTF-8
|
53
|
+
string: <html><body>You are being <a href="https://atnd.org/events/14386">redirected</a>.</body></html>
|
54
|
+
http_version:
|
55
|
+
recorded_at: Sat, 27 Dec 2014 10:29:12 GMT
|
56
|
+
- request:
|
57
|
+
method: get
|
58
|
+
uri: http://atnd.org/events/14386
|
59
|
+
body:
|
60
|
+
encoding: US-ASCII
|
61
|
+
string: ''
|
62
|
+
headers:
|
63
|
+
Accept-Encoding:
|
64
|
+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
|
65
|
+
Accept:
|
66
|
+
- "*/*"
|
67
|
+
User-Agent:
|
68
|
+
- Ruby
|
69
|
+
response:
|
70
|
+
status:
|
71
|
+
code: 302
|
72
|
+
message: Found
|
73
|
+
headers:
|
74
|
+
Date:
|
75
|
+
- Sat, 27 Dec 2014 10:28:38 GMT
|
76
|
+
Server:
|
77
|
+
- Apache
|
78
|
+
X-Frame-Options:
|
79
|
+
- SAMEORIGIN
|
80
|
+
X-Xss-Protection:
|
81
|
+
- 1; mode=block
|
82
|
+
X-Content-Type-Options:
|
83
|
+
- nosniff
|
84
|
+
Cache-Control:
|
85
|
+
- no-cache
|
86
|
+
X-Request-Id:
|
87
|
+
- 0dc8bf12-654e-4486-b856-96eb980dd93f
|
88
|
+
X-Runtime:
|
89
|
+
- '0.005975'
|
90
|
+
X-Powered-By:
|
91
|
+
- Phusion Passenger 4.0.42
|
92
|
+
Set-Cookie:
|
93
|
+
- request_method=GET; path=/
|
94
|
+
Location:
|
95
|
+
- https://atnd.org/events/14386
|
96
|
+
Status:
|
97
|
+
- 302 Found
|
98
|
+
P3p:
|
99
|
+
- CP="NON DSP COR CURa ADMa DEVa CUSo TAIa PSDo OUR BUS UNI COM NAV STA"
|
100
|
+
Transfer-Encoding:
|
101
|
+
- chunked
|
102
|
+
Content-Type:
|
103
|
+
- text/html; charset=utf-8
|
104
|
+
body:
|
105
|
+
encoding: UTF-8
|
106
|
+
string: <html><body>You are being <a href="https://atnd.org/events/14386">redirected</a>.</body></html>
|
107
|
+
http_version:
|
108
|
+
recorded_at: Sat, 27 Dec 2014 10:29:12 GMT
|
109
|
+
recorded_with: VCR 2.9.3
|