spidy 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/spidy/connector.rb +16 -4
- data/lib/spidy/connector/direct.rb +1 -1
- data/lib/spidy/connector/html.rb +1 -1
- data/lib/spidy/connector/json.rb +18 -2
- data/lib/spidy/connector/xml.rb +1 -1
- data/lib/spidy/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44d16509821e1779d821effb7c571d2a11cbb79166168943242628b2e53457ec
|
4
|
+
data.tar.gz: 5b52389a8042c9e69aaeead0d96b81553da2e5f5e51ad16696f87249e38067bd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3475b05ea1235b388960416a03ab96adb64608ce4046d1812b4d80f038a91b99343882e45413ee6a2f74e29626869aca8a48d34f8e57c64b480c8893efcda123
|
7
|
+
data.tar.gz: 7e4fd1abf5898a7a7273d4eaf53021e4c1898d995e8e0d23abfcbf80c3e1b45b2f059c95e48707577bae72a4f158d7cba7f9be097e7e8a277d89e683386c32d6
|
data/Gemfile.lock
CHANGED
data/lib/spidy/connector.rb
CHANGED
@@ -22,11 +22,19 @@ module Spidy::Connector
|
|
22
22
|
'Safari/537.36'
|
23
23
|
].join(' ')
|
24
24
|
|
25
|
+
#
|
26
|
+
# error output logger
|
27
|
+
#
|
28
|
+
DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
|
29
|
+
|
30
|
+
#
|
31
|
+
# static method
|
32
|
+
#
|
25
33
|
module StaticAccessor
|
26
34
|
extend ActiveSupport::Concern
|
27
35
|
class_methods do
|
28
|
-
def call(url, wait_time:
|
29
|
-
new(wait_time: wait_time, user_agent: user_agent).call(url, &block)
|
36
|
+
def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
|
37
|
+
new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
|
30
38
|
end
|
31
39
|
end
|
32
40
|
end
|
@@ -75,10 +83,14 @@ module Spidy::Connector
|
|
75
83
|
#
|
76
84
|
# get connection handller
|
77
85
|
#
|
78
|
-
def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
|
86
|
+
def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
|
79
87
|
return value if value.respond_to?(:call)
|
80
88
|
|
81
|
-
builder = Builder.new(const_get(value.to_s.classify).new(
|
89
|
+
builder = Builder.new(const_get(value.to_s.classify).new(
|
90
|
+
wait_time: wait_time || 5,
|
91
|
+
user_agent: user_agent || USER_AGENT,
|
92
|
+
logger: logger || DEFAULT_LOGGER,
|
93
|
+
), socks_proxy)
|
82
94
|
return builder.origin_connector if socks_proxy.nil? || builder.proxy_disabled?
|
83
95
|
|
84
96
|
builder.proxy_connector
|
data/lib/spidy/connector/html.rb
CHANGED
@@ -8,7 +8,7 @@ class Spidy::Connector::Html
|
|
8
8
|
|
9
9
|
def initialize(wait_time:, user_agent:, logger: nil)
|
10
10
|
@wait_time = wait_time
|
11
|
-
@logger = logger
|
11
|
+
@logger = logger
|
12
12
|
@agent = Mechanize.new
|
13
13
|
@user_agent = user_agent
|
14
14
|
@agent.user_agent = user_agent
|
data/lib/spidy/connector/json.rb
CHANGED
@@ -6,8 +6,12 @@
|
|
6
6
|
class Spidy::Connector::Json
|
7
7
|
include Spidy::Connector::StaticAccessor
|
8
8
|
|
9
|
-
|
9
|
+
attr_reader :logger
|
10
|
+
|
11
|
+
def initialize(wait_time: nil, user_agent: nil, logger: nil)
|
12
|
+
@wait_time = wait_time
|
10
13
|
@user_agent = user_agent
|
14
|
+
@logger = logger
|
11
15
|
end
|
12
16
|
|
13
17
|
def call(url, &block)
|
@@ -15,7 +19,19 @@ class Spidy::Connector::Json
|
|
15
19
|
connect(url, &block)
|
16
20
|
end
|
17
21
|
|
18
|
-
def connect(url)
|
22
|
+
def connect(url, retry_count: 5)
|
19
23
|
OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
|
24
|
+
rescue OpenURI::HTTPError => e
|
25
|
+
logger.call('retry.accessed': Time.current,
|
26
|
+
'retry.uri': url,
|
27
|
+
'retry.response_code': e.message,
|
28
|
+
'retry.rest_count': retry_count)
|
29
|
+
|
30
|
+
retry_count -= 1
|
31
|
+
if retry_count.positive?
|
32
|
+
sleep @wait_time
|
33
|
+
retry
|
34
|
+
end
|
35
|
+
raise e
|
20
36
|
end
|
21
37
|
end
|
data/lib/spidy/connector/xml.rb
CHANGED
data/lib/spidy/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spidy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- aileron
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-09-
|
11
|
+
date: 2020-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|