http_crawler 0.3.0.8 → 0.3.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/http_crawler/client.rb +5 -10
- data/lib/http_crawler/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d9824aa0b465a4488ca1bad737987eae1a2f76b634be2f2aeca8cc0e2a293f0a
|
|
4
|
+
data.tar.gz: cc98362a6f666170acbfe1b20c1543abca5653d6ea2fa0f7b8e1c914c7e8a626
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fa5f84af9447a64a524f8d7dc405427d7cc97123404df09ba7093e2963085a850f109891cf5a6bd654077ba33cf500c33e7f1f00dacaf83bd5023182d9b1d2e3
|
|
7
|
+
data.tar.gz: f79b86da4939e910d6db30f9825431f02246901ed54c671fd686a428b2c6753ad0779e11be6588efb3008bd57d5981f18ba78ceb5cb44f7ef3397b6afb4623cf
|
data/lib/http_crawler/client.rb
CHANGED
|
@@ -150,9 +150,9 @@ module HttpCrawler
|
|
|
150
150
|
|
|
151
151
|
|
|
152
152
|
# 如果自动更新代理 则更新代理返回 true，否则返回false
|
|
153
|
-
def update_proxy?
|
|
153
|
+
def update_proxy?
|
|
154
154
|
if @auto_proxy
|
|
155
|
-
update_proxy
|
|
155
|
+
self.update_proxy
|
|
156
156
|
return true
|
|
157
157
|
else
|
|
158
158
|
return false
|
|
@@ -170,7 +170,7 @@ module HttpCrawler
|
|
|
170
170
|
proxy_r = proxy_client.get_proxy(proxy_params)
|
|
171
171
|
proxy_ip = proxy_r.results unless proxy_r.results.blank?
|
|
172
172
|
if proxy_ip.blank?
|
|
173
|
-
Rails.logger.warn "无最新代理等待5
|
|
173
|
+
Rails.logger.warn "无最新代理等待5秒后重新获取:proxy 为空"
|
|
174
174
|
else
|
|
175
175
|
break
|
|
176
176
|
end
|
|
@@ -179,17 +179,12 @@ module HttpCrawler
|
|
|
179
179
|
|
|
180
180
|
Rails.logger.debug("当前IP => #{@proxy},获取最新代理 => #{proxy_ip}")
|
|
181
181
|
|
|
182
|
-
unless proxy_ip
|
|
183
|
-
Rails.logger.warn "无最新代理等待5
|
|
182
|
+
unless proxy_ip["p_addr"] && proxy_ip["p_port"]
|
|
183
|
+
Rails.logger.warn "无最新代理等待5秒后重新获取:p_addr 或 p_port 为空"
|
|
184
184
|
sleep(5)
|
|
185
185
|
proxy_ip = get_proxy
|
|
186
186
|
end
|
|
187
187
|
|
|
188
|
-
if (@proxy && proxy_ip && @proxy["p_addr"] == proxy_ip["p_addr"] && @proxy["p_port"] == proxy_ip["p_port"])
|
|
189
|
-
Rails.logger.warn "无最新代理等待5秒后重新获取"
|
|
190
|
-
sleep(5)
|
|
191
|
-
proxy_ip = get_proxy
|
|
192
|
-
end
|
|
193
188
|
proxy_ip
|
|
194
189
|
end
|
|
195
190
|
|
data/lib/http_crawler/version.rb
CHANGED