pioneer 0.0.3.alpha → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +6 -1
- data/lib/pioneer/base.rb +3 -0
- data/lib/pioneer/request.rb +9 -3
- data/lib/pioneer/version.rb +1 -1
- metadata +9 -9
data/CHANGELOG
CHANGED
@@ -8,4 +8,9 @@
|
|
8
8
|
|
9
9
|
## v0.0.1alpha
|
10
10
|
|
11
|
-
* Heh, but it is still in dev. So let's call it alpha
|
11
|
+
* Heh, but it is still in dev. So let's call it alpha
|
12
|
+
|
13
|
+
## v0.0.4
|
14
|
+
|
15
|
+
* Fixed stackoverflow on retrying request after failing
|
16
|
+
* added `req.retry` method to retry http request in error handlers
|
data/lib/pioneer/base.rb
CHANGED
@@ -6,6 +6,9 @@ module Pioneer
|
|
6
6
|
class LocationsNotEnumerator < RuntimeError; end
|
7
7
|
class HttpRequestError < RuntimeError; end
|
8
8
|
class HttpResponseError < RuntimeError; end
|
9
|
+
class HttpStatusError < RuntimeError; end
|
10
|
+
class HttpRetryRequest < RuntimeError; end
|
11
|
+
|
9
12
|
class Base
|
10
13
|
attr_reader :name, :concurrency, :sleep, :log_level, :redirect
|
11
14
|
|
data/lib/pioneer/request.rb
CHANGED
@@ -23,12 +23,14 @@ module Pioneer
|
|
23
23
|
@error = "Request totaly failed. Url: #{url}, error: #{e.message}"
|
24
24
|
pioneer.logger.fatal(error)
|
25
25
|
if pioneer.respond_to? :if_request_error
|
26
|
-
return pioneer.
|
26
|
+
return pioneer.if_request_error(self)
|
27
27
|
else
|
28
28
|
raise HttpRequestError, @error
|
29
29
|
end
|
30
30
|
end
|
31
31
|
handle_response_error_or_return_result
|
32
|
+
rescue HttpRetryRequest => e
|
33
|
+
retry
|
32
34
|
end
|
33
35
|
|
34
36
|
# handle http error
|
@@ -37,7 +39,7 @@ module Pioneer
|
|
37
39
|
error = "Response for #{url} get an error: #{response.error}"
|
38
40
|
pioneer.logger.error(error)
|
39
41
|
if pioneer.respond_to? :if_response_error
|
40
|
-
return pioneer.
|
42
|
+
return pioneer.if_response_error(self)
|
41
43
|
else
|
42
44
|
raise HttpResponseError, error
|
43
45
|
end
|
@@ -55,11 +57,15 @@ module Pioneer
|
|
55
57
|
if pioneer.respond_to? "if_status_#{status}".to_sym
|
56
58
|
pioneer.send("if_status_#{status}", self)
|
57
59
|
elsif pioneer.respond_to? :if_status_not_200
|
58
|
-
pioneer.
|
60
|
+
pioneer.if_status_not_200(self)
|
59
61
|
else
|
60
62
|
nil # nothing?
|
61
63
|
end
|
62
64
|
end
|
63
65
|
end
|
66
|
+
|
67
|
+
def retry
|
68
|
+
raise HttpRetryRequest
|
69
|
+
end
|
64
70
|
end
|
65
71
|
end
|
data/lib/pioneer/version.rb
CHANGED
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pioneer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Petr
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-02-
|
12
|
+
date: 2012-02-29 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: yajl-ruby
|
16
|
-
requirement: &
|
16
|
+
requirement: &73303650 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *73303650
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: em-synchrony
|
27
|
-
requirement: &
|
27
|
+
requirement: &73303440 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *73303440
|
36
36
|
description: Simple async HTTP crawler based on em-synchrony
|
37
37
|
email:
|
38
38
|
- pedro.yanoviches@gmail.com
|
@@ -75,9 +75,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
75
75
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
76
76
|
none: false
|
77
77
|
requirements:
|
78
|
-
- - ! '
|
78
|
+
- - ! '>='
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
80
|
+
version: '0'
|
81
81
|
requirements: []
|
82
82
|
rubyforge_project: pioneer
|
83
83
|
rubygems_version: 1.8.15
|