klepto 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +4 -5
- data/lib/klepto/bot.rb +7 -8
- data/lib/klepto/browser.rb +1 -1
- data/lib/klepto/config.rb +3 -3
- data/lib/klepto/version.rb +1 -1
- data/spec/lib/klepto/bot_spec.rb +2 -2
- metadata +11 -11
data/README.md
CHANGED
@@ -171,12 +171,11 @@ end
|
|
171
171
|
## Callbacks & Processing
|
172
172
|
|
173
173
|
* before
|
174
|
-
* :get
|
175
|
-
* :structure
|
174
|
+
* :get (browser, url)
|
176
175
|
* after
|
177
|
-
* :
|
178
|
-
* :get (
|
179
|
-
* :abort (
|
176
|
+
* :structure (Hash) - receives the structure from the page
|
177
|
+
* :get (browser, url) - called after each HTTP GET
|
178
|
+
* :abort (browser, hash(details)) - called after a 4xx or 5xx if config.abort_on_failure is true (default)
|
180
179
|
|
181
180
|
|
182
181
|
## Stuff I'm going to add.
|
data/lib/klepto/bot.rb
CHANGED
@@ -34,9 +34,8 @@ EOS
|
|
34
34
|
@browser.set_headers @config.headers
|
35
35
|
#browser.set_driver config.driver
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
bh.call(url, browser)
|
37
|
+
@config.before_handlers[:get].each { |bh|
|
38
|
+
bh.call(@browser,@config.url)
|
40
39
|
}
|
41
40
|
|
42
41
|
begin
|
@@ -44,18 +43,18 @@ EOS
|
|
44
43
|
|
45
44
|
# Fire callbacks on GET
|
46
45
|
@config.after_handlers[:get].each do |ah|
|
47
|
-
ah.call(@browser
|
46
|
+
ah.call(@browser, @config.url)
|
48
47
|
end
|
49
48
|
|
50
49
|
# Dispatch all the handlers for HTTP Status Codes.
|
51
50
|
@browser.statuses.each do |status|
|
52
|
-
@config.dispatch_status_handlers(status, @browser
|
51
|
+
@config.dispatch_status_handlers(status, @browser)
|
53
52
|
end
|
54
53
|
|
55
54
|
# If the page was not a failure or if not aborting, structure that bad boy.
|
56
55
|
if (@browser.failure? && @config.abort_on_failure?) || (@config.abort_on_redirect? && @browser.was_redirected?)
|
57
56
|
@config.after_handlers[:abort].each do |ah|
|
58
|
-
ah.call(@browser
|
57
|
+
ah.call(@browser,{
|
59
58
|
browser_failure: @browser.failure?,
|
60
59
|
abort_on_failure: @config.abort_on_failure?,
|
61
60
|
abort_on_redirect: @config.abort_on_redirect?,
|
@@ -66,7 +65,7 @@ EOS
|
|
66
65
|
@structure = __structure(@browser.page)
|
67
66
|
end
|
68
67
|
rescue Capybara::Poltergeist::TimeoutError => ex
|
69
|
-
config.dispatch_timeout_handler(ex, @config.url)
|
68
|
+
config.dispatch_timeout_handler(ex, @browser, @config.url)
|
70
69
|
end
|
71
70
|
|
72
71
|
@structure
|
@@ -85,7 +84,7 @@ EOS
|
|
85
84
|
end
|
86
85
|
|
87
86
|
# Call after(:each) handlers...
|
88
|
-
config.after_handlers[:
|
87
|
+
config.after_handlers[:structure].each { |ah| ah.call(structure._hash) }
|
89
88
|
|
90
89
|
structure._hash
|
91
90
|
end
|
data/lib/klepto/browser.rb
CHANGED
data/lib/klepto/config.rb
CHANGED
@@ -8,11 +8,11 @@ module Klepto
|
|
8
8
|
@abort_on_failure = true
|
9
9
|
@abort_on_redirect = false
|
10
10
|
@after_handlers = {
|
11
|
-
:
|
11
|
+
:structure => [], #after each call to
|
12
12
|
:get => [], #after GET, before structure
|
13
13
|
:abort=> [] #after abort
|
14
14
|
}
|
15
|
-
@before_handlers = {:
|
15
|
+
@before_handlers = {:get => []}
|
16
16
|
@status_handlers = {}
|
17
17
|
@handlers = {}
|
18
18
|
end
|
@@ -85,7 +85,7 @@ module Klepto
|
|
85
85
|
end
|
86
86
|
end
|
87
87
|
|
88
|
-
def after(which = :
|
88
|
+
def after(which = :structure, &block)
|
89
89
|
@after_handlers[which] ||= []
|
90
90
|
@after_handlers[which].push block
|
91
91
|
end
|
data/lib/klepto/version.rb
CHANGED
data/spec/lib/klepto/bot_spec.rb
CHANGED
@@ -137,7 +137,7 @@ describe Klepto::Bot do
|
|
137
137
|
StatusLog.create message: '200'
|
138
138
|
}
|
139
139
|
|
140
|
-
config.after(:
|
140
|
+
config.after(:structure) do |resource|
|
141
141
|
@user = User.new
|
142
142
|
@user.name = resource[:name]
|
143
143
|
@user.username = resource[:username]
|
@@ -255,7 +255,7 @@ describe Klepto::Bot do
|
|
255
255
|
permalink '.time a', :css, :attr => :href
|
256
256
|
end
|
257
257
|
|
258
|
-
config.after(:
|
258
|
+
config.after(:structure) do |resource|
|
259
259
|
@user = User.new
|
260
260
|
@user.name = resource[:name]
|
261
261
|
@user.username = resource[:username]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: klepto
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -13,7 +13,7 @@ date: 2013-06-01 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: poltergeist
|
16
|
-
requirement: &
|
16
|
+
requirement: &70151609865620 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - =
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 1.1.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70151609865620
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: capybara
|
27
|
-
requirement: &
|
27
|
+
requirement: &70151609865120 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - =
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 2.0.2
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70151609865120
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: nokogiri
|
38
|
-
requirement: &
|
38
|
+
requirement: &70151609864660 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 1.5.6
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70151609864660
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activesupport
|
49
|
-
requirement: &
|
49
|
+
requirement: &70151609864280 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70151609864280
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: multi_json
|
60
|
-
requirement: &
|
60
|
+
requirement: &70151609863740 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: '1.0'
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70151609863740
|
69
69
|
description: Tearing up web pages into ActiveRecord resources
|
70
70
|
email:
|
71
71
|
- github@coryodaniel.com
|