http_crawler 0.3.1.28 → 0.3.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/http_crawler/client.rb +5 -3
- data/lib/http_crawler/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 329e17b63dd792550707d01ce3a310a8bc81d2b04c5d2a6b5efc983a1b0a1eff
|
|
4
|
+
data.tar.gz: 1a0d14ccb1326181e9c929de5ce1f8e057c698c3b10ee88eab1a9685b6bf0c39
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 88a8389db13b3ed17a82772d9a66378c6184133636d53617e5550d6cdd27cd542c3a5ca1320faf94c3b1a2a20441b119e1a9b915ecd268802e0a993711149fcc
|
|
7
|
+
data.tar.gz: 8ac49deb0e1b0c3a813f2648d7d9a55e6135b883d6f9c73c7d962c37f5af364d79284c58e5069555bef433c84e4e91d71904ca3b695aa28daef2e503ee23625f
|
data/lib/http_crawler/client.rb
CHANGED
|
@@ -61,7 +61,7 @@ module HttpCrawler
|
|
|
61
61
|
attr_accessor :max_error_num
|
|
62
62
|
# 最大错误重试次数
|
|
63
63
|
def max_error_num
|
|
64
|
-
@max_error_num ||=
|
|
64
|
+
@max_error_num ||= 2
|
|
65
65
|
end
|
|
66
66
|
|
|
67
67
|
attr_reader :uri
|
|
@@ -206,6 +206,8 @@ module HttpCrawler
|
|
|
206
206
|
proxy_client = HttpCrawler::Proxy.for(proxy_api)
|
|
207
207
|
proxy_r = proxy_client.get_proxy(proxy_params.symbolize_keys)
|
|
208
208
|
proxy_ip = proxy_r.results unless proxy_r.results.blank?
|
|
209
|
+
# 测试本地代理
|
|
210
|
+
# proxy_ip = {p_addr: "127.0.0.1", p_port: 8888} if "production" =! Rails.env
|
|
209
211
|
if proxy_ip.blank?
|
|
210
212
|
Rails.logger.warn "无最新代理等待5秒后重新获取:proxy 为空"
|
|
211
213
|
else
|
|
@@ -289,7 +291,7 @@ module HttpCrawler
|
|
|
289
291
|
end
|
|
290
292
|
|
|
291
293
|
# 发送 post 请求
|
|
292
|
-
def post(path, params = {},format = :form)
|
|
294
|
+
def post(path, params = {}, format = :form)
|
|
293
295
|
raise "Client uri为空" unless self.uri
|
|
294
296
|
request {http.post((self.uri + path).to_s, format => params, :ssl_context => @ctx)}
|
|
295
297
|
end
|
|
@@ -334,7 +336,7 @@ module HttpCrawler
|
|
|
334
336
|
else
|
|
335
337
|
# 每次错误次数尝试 -1
|
|
336
338
|
n -= 1
|
|
337
|
-
self.update_proxy?
|
|
339
|
+
# self.update_proxy?
|
|
338
340
|
retry
|
|
339
341
|
end
|
|
340
342
|
end
|
data/lib/http_crawler/version.rb
CHANGED