nethttputils 0.4.1.0 → 0.4.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/nethttputils.rb +18 -13
- data/nethttputils.gemspec +2 -2
- metadata +2 -5
- data/.travis.yml +0 -29
- data/Gemfile +0 -3
- data/Rakefile +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c68a49af2dbbca3cacf81cd5518df2ce6f2cb72
|
4
|
+
data.tar.gz: bae1db040743d7f9ea9ac17d7aeafba778f227c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ba54aa8a5fe007911b8af3117061b1a279182eb27cbdcd2be0a6293428d4408356b2b8238c7342284e93da9ee20f62d169b79346f1f30019677c24c0ca5c9af
|
7
|
+
data.tar.gz: 23e5657489821b849a611a3692d14335b46c1f367dfff1c130d862a1390eb5234742110cdda9d69bb06f9206ae98ed0fdf6dfa6d66fc7056bf45dacd62a0ca13
|
data/lib/nethttputils.rb
CHANGED
@@ -33,7 +33,7 @@ module NetHTTPUtils
|
|
33
33
|
gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
|
34
34
|
end
|
35
35
|
|
36
|
-
def start_http url, max_start_http_retry_delay = 3600, timeout = 30
|
36
|
+
def start_http url, max_start_http_retry_delay = 3600, timeout = 30, proxy = nil
|
37
37
|
uri = url
|
38
38
|
uri = URI.parse begin
|
39
39
|
URI url
|
@@ -45,6 +45,7 @@ module NetHTTPUtils
|
|
45
45
|
begin
|
46
46
|
Net::HTTP.start(
|
47
47
|
uri.host, uri.port,
|
48
|
+
*(proxy.split ?: if proxy),
|
48
49
|
use_ssl: uri.scheme == "https",
|
49
50
|
verify_mode: OpenSSL::SSL::VERIFY_NONE,
|
50
51
|
**({open_timeout: timeout}), # if timeout
|
@@ -66,7 +67,7 @@ module NetHTTPUtils
|
|
66
67
|
end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
|
67
68
|
http
|
68
69
|
end
|
69
|
-
rescue Errno::ECONNREFUSED => e
|
70
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
|
70
71
|
if max_start_http_retry_delay < delay *= 2
|
71
72
|
e.message.concat " to #{uri}"
|
72
73
|
raise
|
@@ -74,10 +75,6 @@ module NetHTTPUtils
|
|
74
75
|
logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
|
75
76
|
sleep delay
|
76
77
|
retry
|
77
|
-
rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
|
78
|
-
logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
|
79
|
-
sleep 5
|
80
|
-
retry
|
81
78
|
rescue SocketError => e
|
82
79
|
if max_start_http_retry_delay < delay *= 2
|
83
80
|
e.message.concat " to #{uri}"
|
@@ -172,7 +169,7 @@ module NetHTTPUtils
|
|
172
169
|
delay = 5
|
173
170
|
response = begin
|
174
171
|
http.request request, &block
|
175
|
-
rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
|
172
|
+
rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
|
176
173
|
raise if max_read_retry_delay < delay *= 2
|
177
174
|
logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
|
178
175
|
sleep delay
|
@@ -186,6 +183,7 @@ module NetHTTPUtils
|
|
186
183
|
# response.instance_variable_set "@nethttputils_close", http.method(:finish)
|
187
184
|
# response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
|
188
185
|
|
186
|
+
now = Time.now
|
189
187
|
remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
|
190
188
|
logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
|
191
189
|
[
|
@@ -197,7 +195,13 @@ module NetHTTPUtils
|
|
197
195
|
[
|
198
196
|
response.fetch("x-rate-limit-remaining").to_i,
|
199
197
|
response.fetch("x-rate-limit-reset").to_i,
|
200
|
-
|
198
|
+
now.to_i,
|
199
|
+
]
|
200
|
+
elsif response.key? "x-ratelimit-remaining"
|
201
|
+
[
|
202
|
+
response.fetch("x-ratelimit-remaining").to_i,
|
203
|
+
now + response.fetch("x-ratelimit-reset").to_i,
|
204
|
+
now.to_i,
|
201
205
|
]
|
202
206
|
end
|
203
207
|
if remaining
|
@@ -292,11 +296,12 @@ module NetHTTPUtils
|
|
292
296
|
|
293
297
|
require "set"
|
294
298
|
@@_405 ||= Set.new
|
295
|
-
def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil,
|
299
|
+
def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil,
|
300
|
+
timeout: 30,
|
296
301
|
max_start_http_retry_delay: 3600,
|
297
302
|
max_read_retry_delay: 3600,
|
298
303
|
patch_request: nil, &block
|
299
|
-
http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
|
304
|
+
http = start_http http, max_start_http_retry_delay, timeout, *proxy unless http.is_a? Net::HTTP
|
300
305
|
path = http.instance_variable_get(:@uri).path
|
301
306
|
|
302
307
|
check_code = lambda do |body|
|
@@ -479,13 +484,13 @@ if $0 == __FILE__
|
|
479
484
|
fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
|
480
485
|
fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
|
481
486
|
[
|
482
|
-
["https://imgur.com/a/
|
483
|
-
["https://imgur.com/mM4Dh7Z"],
|
487
|
+
# ["https://imgur.com/a/oacI3gl"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
|
488
|
+
# ["https://imgur.com/mM4Dh7Z"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
|
484
489
|
["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
|
485
490
|
].each do |url, expectation|
|
486
491
|
begin
|
487
492
|
puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
|
488
|
-
fail
|
493
|
+
fail url
|
489
494
|
rescue NetHTTPUtils::Error => e
|
490
495
|
raise e.code.inspect unless e.code == 404
|
491
496
|
raise e.to_s if e.to_s != expectation if expectation
|
data/nethttputils.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "nethttputils"
|
3
|
-
spec.version = "0.4.1.
|
3
|
+
spec.version = "0.4.1.1"
|
4
4
|
spec.summary = "this tool is like a pet that I adopted young and now I depend on, sorry"
|
5
5
|
spec.description = <<-EOF
|
6
6
|
Back in 2015 I was a guy automating things at my job and two scripts had a common need --
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.license = "MIT"
|
22
22
|
|
23
23
|
spec.require_path = "lib"
|
24
|
-
spec.files =
|
24
|
+
spec.files = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
|
25
25
|
|
26
26
|
spec.add_dependency "addressable"
|
27
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nethttputils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.1.
|
4
|
+
version: 0.4.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Maslov aka Nakilon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -40,10 +40,7 @@ executables: []
|
|
40
40
|
extensions: []
|
41
41
|
extra_rdoc_files: []
|
42
42
|
files:
|
43
|
-
- ".travis.yml"
|
44
|
-
- Gemfile
|
45
43
|
- LICENSE
|
46
|
-
- Rakefile
|
47
44
|
- lib/nethttputils.rb
|
48
45
|
- nethttputils.gemspec
|
49
46
|
homepage: https://github.com/nakilon/nethttputils
|
data/.travis.yml
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
---
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
script: "bundle install && bundle exec ruby lib/nethttputils.rb"
|
5
|
-
|
6
|
-
os:
|
7
|
-
- linux
|
8
|
-
- osx
|
9
|
-
rvm:
|
10
|
-
- ruby-head
|
11
|
-
- 2.5
|
12
|
-
- 2.4
|
13
|
-
- 2.3
|
14
|
-
- 2.2
|
15
|
-
- 2.1
|
16
|
-
- 2.0
|
17
|
-
- jruby-head
|
18
|
-
matrix:
|
19
|
-
allow_failures:
|
20
|
-
# something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
|
21
|
-
- rvm: jruby-head
|
22
|
-
- rvm: 2.3
|
23
|
-
os: osx
|
24
|
-
- rvm: 2.2
|
25
|
-
os: osx
|
26
|
-
- rvm: 2.1
|
27
|
-
os: osx
|
28
|
-
- rvm: 2.0
|
29
|
-
os: osx
|
data/Gemfile
DELETED
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "bundler/gem_tasks"
|