klepto 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/klepto/bot.rb +12 -0
- data/lib/klepto/browser.rb +12 -2
- data/lib/klepto/config.rb +5 -0
- data/lib/klepto/version.rb +1 -1
- data/spec/lib/klepto/browser_spec.rb +4 -0
- data/spec/lib/klepto/config_spec.rb +5 -0
- metadata +11 -11
data/lib/klepto/bot.rb
CHANGED
@@ -1,6 +1,17 @@
|
|
1
1
|
module Klepto
|
2
2
|
class Bot
|
3
3
|
attr_reader :config
|
4
|
+
@@_bots = {}
|
5
|
+
class << self
|
6
|
+
def run(name,*urls)
|
7
|
+
urls.each do |url|
|
8
|
+
@@_bots[name].parse! url
|
9
|
+
end
|
10
|
+
end
|
11
|
+
def make(name, &block)
|
12
|
+
@@_bots[name] = Klepto::Bot.new(&block)
|
13
|
+
end
|
14
|
+
end
|
4
15
|
|
5
16
|
def initialize(*urls, &block)
|
6
17
|
@config = Klepto::Config.new
|
@@ -33,6 +44,7 @@ EOS
|
|
33
44
|
browser = Klepto::Browser.new
|
34
45
|
|
35
46
|
browser.set_headers config.headers
|
47
|
+
browser.set_driver config.driver
|
36
48
|
|
37
49
|
begin
|
38
50
|
browser.fetch! url
|
data/lib/klepto/browser.rb
CHANGED
@@ -6,6 +6,14 @@ module Klepto
|
|
6
6
|
super
|
7
7
|
end
|
8
8
|
|
9
|
+
def set_driver(use_driver)
|
10
|
+
@use_driver = use_driver
|
11
|
+
end
|
12
|
+
|
13
|
+
def use_driver
|
14
|
+
@use_driver || :poltergeist
|
15
|
+
end
|
16
|
+
|
9
17
|
def set_headers(headers)
|
10
18
|
page.driver.headers = headers
|
11
19
|
end
|
@@ -27,8 +35,10 @@ module Klepto
|
|
27
35
|
end
|
28
36
|
|
29
37
|
def fetch!(url)
|
30
|
-
|
31
|
-
|
38
|
+
Capybara.using_driver use_driver do
|
39
|
+
visit url
|
40
|
+
page
|
41
|
+
end
|
32
42
|
end
|
33
43
|
end
|
34
44
|
end
|
data/lib/klepto/config.rb
CHANGED
@@ -24,6 +24,11 @@ module Klepto
|
|
24
24
|
# @handlers[group][handler].each{|handler| handler.call(*args)}
|
25
25
|
# end
|
26
26
|
|
27
|
+
def driver(_driver=nil)
|
28
|
+
@default_driver = _driver if _driver
|
29
|
+
@default_driver
|
30
|
+
end
|
31
|
+
|
27
32
|
def headers(_headers=nil)
|
28
33
|
@headers = _headers if _headers
|
29
34
|
@headers
|
data/lib/klepto/version.rb
CHANGED
@@ -12,4 +12,8 @@ describe Klepto::Browser, :vcr => {:record => :new_episodes} do
|
|
12
12
|
@page = @browser.fetch! 'http://www.example.com'
|
13
13
|
@page.status_code.should be(200)
|
14
14
|
end
|
15
|
+
|
16
|
+
it 'should use poltergeist by default' do
|
17
|
+
@browser.use_driver.should == :poltergeist
|
18
|
+
end
|
15
19
|
end
|
@@ -14,6 +14,7 @@ describe Klepto::Config do
|
|
14
14
|
@config.on_http_status('5xx','4xx'){
|
15
15
|
"Its crazy."
|
16
16
|
}
|
17
|
+
@config.driver :cool_driver
|
17
18
|
@config.abort_on_failure(false)
|
18
19
|
end
|
19
20
|
|
@@ -21,6 +22,10 @@ describe Klepto::Config do
|
|
21
22
|
@config.headers['Referer'].should eq('http://example.com')
|
22
23
|
end
|
23
24
|
|
25
|
+
it 'should default to poltergeist as the driver' do
|
26
|
+
@config.driver.should == :cool_driver
|
27
|
+
end
|
28
|
+
|
24
29
|
it 'should have a 2xx status handler' do
|
25
30
|
@config.instance_variable_get("@status_handlers")['2xx'].first.call.should eq ('Its 2xx')
|
26
31
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: klepto
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -13,7 +13,7 @@ date: 2013-05-28 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: poltergeist
|
16
|
-
requirement: &
|
16
|
+
requirement: &70226097145120 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - =
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 1.1.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70226097145120
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: capybara
|
27
|
-
requirement: &
|
27
|
+
requirement: &70226097144620 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - =
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 2.0.2
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70226097144620
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: nokogiri
|
38
|
-
requirement: &
|
38
|
+
requirement: &70226097144160 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 1.5.6
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70226097144160
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activesupport
|
49
|
-
requirement: &
|
49
|
+
requirement: &70226097143780 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70226097143780
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: multi_json
|
60
|
-
requirement: &
|
60
|
+
requirement: &70226097143240 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: '1.0'
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70226097143240
|
69
69
|
description: Tearing up web pages into ActiveRecord resources
|
70
70
|
email:
|
71
71
|
- github@coryodaniel.com
|