klepto 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/klepto/bot.rb +7 -2
- data/lib/klepto/config.rb +20 -2
- data/lib/klepto/version.rb +1 -1
- metadata +11 -11
data/lib/klepto/bot.rb
CHANGED
|
@@ -33,8 +33,13 @@ EOS
|
|
|
33
33
|
browser = Klepto::Browser.new
|
|
34
34
|
|
|
35
35
|
browser.set_headers config.headers
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
|
|
37
|
+
begin
|
|
38
|
+
browser.fetch! url
|
|
39
|
+
rescue Capybara::Poltergeist::TimeoutError => ex
|
|
40
|
+
dispatch_timeout_handler(ex, url)
|
|
41
|
+
end
|
|
42
|
+
|
|
38
43
|
# Fire callbacks on GET
|
|
39
44
|
config.after_handlers[:get].each do |ah|
|
|
40
45
|
ah.call(browser.page)
|
data/lib/klepto/config.rb
CHANGED
|
@@ -29,6 +29,10 @@ module Klepto
|
|
|
29
29
|
@headers
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
+
def has_timeout_handler?
|
|
33
|
+
@status_handlers[:timeout] && @status_handlers[:timeout].any?
|
|
34
|
+
end
|
|
35
|
+
|
|
32
36
|
def abort_on_failure?
|
|
33
37
|
!!@abort_on_failure
|
|
34
38
|
end
|
|
@@ -46,6 +50,21 @@ module Klepto
|
|
|
46
50
|
@abort_on_redirect = aor
|
|
47
51
|
end
|
|
48
52
|
|
|
53
|
+
def on_http_timeout(&block)
|
|
54
|
+
@status_handlers[:timeout] ||= []
|
|
55
|
+
@status_handlers[:timeout].push block
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def dispatch_timeout_handler(ex, url)
|
|
59
|
+
if @status_handlers[:timeout]
|
|
60
|
+
@status_handlers[:timeout].each do |handler|
|
|
61
|
+
handler.call(ex, url)
|
|
62
|
+
end
|
|
63
|
+
else
|
|
64
|
+
raise ex
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
49
68
|
def on_http_status(*statuses,&block)
|
|
50
69
|
statuses.each do |status|
|
|
51
70
|
@status_handlers[status] ||= []
|
|
@@ -54,8 +73,7 @@ module Klepto
|
|
|
54
73
|
end
|
|
55
74
|
|
|
56
75
|
def dispatch_status_handlers(status, page)
|
|
57
|
-
|
|
58
|
-
if handlers.present?
|
|
76
|
+
if @status_handlers[status]
|
|
59
77
|
@status_handlers[status].each do |handler|
|
|
60
78
|
handler.call(page)
|
|
61
79
|
end
|
data/lib/klepto/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: klepto
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.3
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2013-05-28 00:00:00.000000000 Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: poltergeist
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70174422733860 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - =
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: 1.1.0
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70174422733860
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: capybara
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70174422733120 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - =
|
|
@@ -32,10 +32,10 @@ dependencies:
|
|
|
32
32
|
version: 2.0.2
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70174422733120
|
|
36
36
|
- !ruby/object:Gem::Dependency
|
|
37
37
|
name: nokogiri
|
|
38
|
-
requirement: &
|
|
38
|
+
requirement: &70174422732120 !ruby/object:Gem::Requirement
|
|
39
39
|
none: false
|
|
40
40
|
requirements:
|
|
41
41
|
- - ~>
|
|
@@ -43,10 +43,10 @@ dependencies:
|
|
|
43
43
|
version: 1.5.6
|
|
44
44
|
type: :runtime
|
|
45
45
|
prerelease: false
|
|
46
|
-
version_requirements: *
|
|
46
|
+
version_requirements: *70174422732120
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: activesupport
|
|
49
|
-
requirement: &
|
|
49
|
+
requirement: &70174422731560 !ruby/object:Gem::Requirement
|
|
50
50
|
none: false
|
|
51
51
|
requirements:
|
|
52
52
|
- - ! '>='
|
|
@@ -54,10 +54,10 @@ dependencies:
|
|
|
54
54
|
version: '0'
|
|
55
55
|
type: :runtime
|
|
56
56
|
prerelease: false
|
|
57
|
-
version_requirements: *
|
|
57
|
+
version_requirements: *70174422731560
|
|
58
58
|
- !ruby/object:Gem::Dependency
|
|
59
59
|
name: multi_json
|
|
60
|
-
requirement: &
|
|
60
|
+
requirement: &70174422729660 !ruby/object:Gem::Requirement
|
|
61
61
|
none: false
|
|
62
62
|
requirements:
|
|
63
63
|
- - ~>
|
|
@@ -65,7 +65,7 @@ dependencies:
|
|
|
65
65
|
version: '1.0'
|
|
66
66
|
type: :runtime
|
|
67
67
|
prerelease: false
|
|
68
|
-
version_requirements: *
|
|
68
|
+
version_requirements: *70174422729660
|
|
69
69
|
description: Tearing up web pages into ActiveRecord resources
|
|
70
70
|
email:
|
|
71
71
|
- github@coryodaniel.com
|