nethttputils 0.4.1.0 → 0.4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/nethttputils.rb +50 -32
- data/nethttputils.gemspec +2 -2
- metadata +2 -5
- data/.travis.yml +0 -29
- data/Gemfile +0 -3
- data/Rakefile +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6b310f69be9a582878a60741eb385b2cbc61884b
|
4
|
+
data.tar.gz: fee129582c51e0ec02abc63e318a681534edc391
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7f628d6558dbe9aa0d08125c5a294c957235858829013ac4dd655b0b28fc63e38bed1827ce35fbb9de1d66ca6acde828fa3a5d77ea90a3e08238751cab9b0ea
|
7
|
+
data.tar.gz: ea174acf8e166ef034d6d151b171babdb19fa3736cd23fbdb006c460deb28236f1ce33c1bbc72194d30695498ccc1292108063b6319b54eace9631a5b3e68bc7
|
data/lib/nethttputils.rb
CHANGED
@@ -33,7 +33,8 @@ module NetHTTPUtils
|
|
33
33
|
gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
|
34
34
|
end
|
35
35
|
|
36
|
-
def start_http url, max_start_http_retry_delay = 3600, timeout =
|
36
|
+
def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
|
37
|
+
timeout ||= 30
|
37
38
|
uri = url
|
38
39
|
uri = URI.parse begin
|
39
40
|
URI url
|
@@ -45,6 +46,7 @@ module NetHTTPUtils
|
|
45
46
|
begin
|
46
47
|
Net::HTTP.start(
|
47
48
|
uri.host, uri.port,
|
49
|
+
*(proxy.split ?: if proxy),
|
48
50
|
use_ssl: uri.scheme == "https",
|
49
51
|
verify_mode: OpenSSL::SSL::VERIFY_NONE,
|
50
52
|
**({open_timeout: timeout}), # if timeout
|
@@ -66,7 +68,7 @@ module NetHTTPUtils
|
|
66
68
|
end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
|
67
69
|
http
|
68
70
|
end
|
69
|
-
rescue Errno::ECONNREFUSED => e
|
71
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
|
70
72
|
if max_start_http_retry_delay < delay *= 2
|
71
73
|
e.message.concat " to #{uri}"
|
72
74
|
raise
|
@@ -74,10 +76,6 @@ module NetHTTPUtils
|
|
74
76
|
logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
|
75
77
|
sleep delay
|
76
78
|
retry
|
77
|
-
rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
|
78
|
-
logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
|
79
|
-
sleep 5
|
80
|
-
retry
|
81
79
|
rescue SocketError => e
|
82
80
|
if max_start_http_retry_delay < delay *= 2
|
83
81
|
e.message.concat " to #{uri}"
|
@@ -103,14 +101,17 @@ module NetHTTPUtils
|
|
103
101
|
end
|
104
102
|
|
105
103
|
private
|
106
|
-
def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout:
|
107
|
-
|
104
|
+
def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, force_post: false, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
|
105
|
+
timeout ||= 30
|
106
|
+
logger = NetHTTPUtils.logger
|
107
|
+
logger.info [mtd, http].inspect
|
108
108
|
|
109
109
|
uri = http.instance_variable_get :@uri
|
110
|
-
|
110
|
+
if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty? # not .upcase! because it's not defined for Symbol
|
111
|
+
logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query
|
111
112
|
# we can't just merge because URI fails to parse such queries as "/?1"
|
112
|
-
|
113
|
-
|
113
|
+
uri.query = URI.encode_www_form form
|
114
|
+
end
|
114
115
|
cookies = {}
|
115
116
|
prepare_request = lambda do |uri|
|
116
117
|
case mtd.upcase
|
@@ -129,15 +130,20 @@ module NetHTTPUtils
|
|
129
130
|
request.basic_auth *auth if auth
|
130
131
|
if (mtd == :POST || mtd == :PATCH) && !form.empty?
|
131
132
|
case type
|
132
|
-
when :json
|
133
|
-
|
134
|
-
|
133
|
+
when :json
|
134
|
+
request.body = JSON.dump form
|
135
|
+
request.content_type = "application/json"
|
136
|
+
when :multipart
|
137
|
+
request.set_form form, "multipart/form-data"
|
138
|
+
when :form
|
139
|
+
if form.any?{ |k, v| v.respond_to? :to_path }
|
135
140
|
request.set_form form, "multipart/form-data"
|
136
141
|
else
|
137
142
|
request.set_form_data form
|
138
143
|
request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
|
139
144
|
end
|
140
|
-
else
|
145
|
+
else
|
146
|
+
raise "unknown content-type '#{type}'"
|
141
147
|
end
|
142
148
|
end
|
143
149
|
header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
|
@@ -172,7 +178,7 @@ module NetHTTPUtils
|
|
172
178
|
delay = 5
|
173
179
|
response = begin
|
174
180
|
http.request request, &block
|
175
|
-
rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
|
181
|
+
rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
|
176
182
|
raise if max_read_retry_delay < delay *= 2
|
177
183
|
logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
|
178
184
|
sleep delay
|
@@ -180,12 +186,13 @@ module NetHTTPUtils
|
|
180
186
|
rescue EOFError => e
|
181
187
|
raise unless e.backtrace.empty?
|
182
188
|
# https://bugs.ruby-lang.org/issues/13018
|
183
|
-
# https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
|
189
|
+
# https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
|
184
190
|
raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
|
185
191
|
end
|
186
192
|
# response.instance_variable_set "@nethttputils_close", http.method(:finish)
|
187
193
|
# response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
|
188
194
|
|
195
|
+
now = Time.now
|
189
196
|
remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
|
190
197
|
logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
|
191
198
|
[
|
@@ -197,7 +204,13 @@ module NetHTTPUtils
|
|
197
204
|
[
|
198
205
|
response.fetch("x-rate-limit-remaining").to_i,
|
199
206
|
response.fetch("x-rate-limit-reset").to_i,
|
200
|
-
|
207
|
+
now.to_i,
|
208
|
+
]
|
209
|
+
elsif response.key? "x-ratelimit-remaining"
|
210
|
+
[
|
211
|
+
response.fetch("x-ratelimit-remaining").to_i,
|
212
|
+
now + response.fetch("x-ratelimit-reset").to_i,
|
213
|
+
now.to_i,
|
201
214
|
]
|
202
215
|
end
|
203
216
|
if remaining
|
@@ -222,8 +235,12 @@ module NetHTTPUtils
|
|
222
235
|
response.add_field "Set-Cookie", "#{k}=#{v}"
|
223
236
|
end
|
224
237
|
|
238
|
+
logger.info "response.code = #{response.code}"
|
225
239
|
case response.code
|
240
|
+
when /\A20/
|
241
|
+
response
|
226
242
|
when /\A30\d\z/
|
243
|
+
next response if no_redirect
|
227
244
|
logger.info "redirect: #{response["location"]}"
|
228
245
|
require "addressable"
|
229
246
|
new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
|
@@ -234,10 +251,10 @@ module NetHTTPUtils
|
|
234
251
|
http.use_ssl? != (new_uri.scheme == "https")
|
235
252
|
logger.debug "changing host from '#{http.address}' to '#{new_host}'"
|
236
253
|
# http.finish # why commented out?
|
237
|
-
http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
|
254
|
+
http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
|
238
255
|
end
|
239
|
-
if request.method == "POST"
|
240
|
-
logger.info "POST redirects to GET (RFC)"
|
256
|
+
if !force_post && request.method == "POST"
|
257
|
+
logger.info "POST redirects to GET (RFC)" # TODO: do it only on code 307; note that some servers still do 302
|
241
258
|
mtd = :GET
|
242
259
|
end
|
243
260
|
do_request.call prepare_request[new_uri]
|
@@ -270,11 +287,10 @@ module NetHTTPUtils
|
|
270
287
|
end
|
271
288
|
}"
|
272
289
|
response
|
273
|
-
when /\A20/
|
274
|
-
response
|
275
290
|
else
|
276
|
-
logger.warn "code #{response.code}
|
277
|
-
|
291
|
+
logger.warn "code #{response.code} from #{request.method} #{request.uri} at #{
|
292
|
+
caller_path, caller_locs = caller_locations.chunk(&:path).first
|
293
|
+
[caller_path, caller_locs.map(&:lineno).chunk(&:itself).map(&:first)].join ":"
|
278
294
|
}"
|
279
295
|
logger.debug "< body: #{
|
280
296
|
response.body.tap do |body|
|
@@ -292,11 +308,13 @@ module NetHTTPUtils
|
|
292
308
|
|
293
309
|
require "set"
|
294
310
|
@@_405 ||= Set.new
|
295
|
-
def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil,
|
311
|
+
def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false,
|
312
|
+
timeout: nil, no_redirect: false,
|
296
313
|
max_start_http_retry_delay: 3600,
|
297
314
|
max_read_retry_delay: 3600,
|
298
315
|
patch_request: nil, &block
|
299
|
-
|
316
|
+
timeout ||= 30
|
317
|
+
http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
|
300
318
|
path = http.instance_variable_get(:@uri).path
|
301
319
|
|
302
320
|
check_code = lambda do |body|
|
@@ -327,7 +345,8 @@ module NetHTTPUtils
|
|
327
345
|
check_code.call body
|
328
346
|
end
|
329
347
|
end
|
330
|
-
body = read http, mtd, type, form: form, header: header, auth: auth,
|
348
|
+
body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
|
349
|
+
timeout: timeout, no_redirect: no_redirect,
|
331
350
|
max_read_retry_delay: max_read_retry_delay,
|
332
351
|
patch_request: patch_request, &block
|
333
352
|
check_code.call body
|
@@ -337,7 +356,6 @@ module NetHTTPUtils
|
|
337
356
|
Zlib::GzipReader.new(StringIO.new(body)).read
|
338
357
|
else
|
339
358
|
body
|
340
|
-
end.tap do |string|
|
341
359
|
end
|
342
360
|
# ensure
|
343
361
|
# response.instance_variable_get("@nethttputils_close").call if response
|
@@ -479,13 +497,13 @@ if $0 == __FILE__
|
|
479
497
|
fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
|
480
498
|
fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
|
481
499
|
[
|
482
|
-
["https://imgur.com/a/
|
483
|
-
["https://imgur.com/mM4Dh7Z"],
|
500
|
+
# ["https://imgur.com/a/oacI3gl"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
|
501
|
+
# ["https://imgur.com/mM4Dh7Z"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
|
484
502
|
["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
|
485
503
|
].each do |url, expectation|
|
486
504
|
begin
|
487
505
|
puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
|
488
|
-
fail
|
506
|
+
fail url
|
489
507
|
rescue NetHTTPUtils::Error => e
|
490
508
|
raise e.code.inspect unless e.code == 404
|
491
509
|
raise e.to_s if e.to_s != expectation if expectation
|
data/nethttputils.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "nethttputils"
|
3
|
-
spec.version = "0.4.
|
3
|
+
spec.version = "0.4.2.0"
|
4
4
|
spec.summary = "this tool is like a pet that I adopted young and now I depend on, sorry"
|
5
5
|
spec.description = <<-EOF
|
6
6
|
Back in 2015 I was a guy automating things at my job and two scripts had a common need --
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.license = "MIT"
|
22
22
|
|
23
23
|
spec.require_path = "lib"
|
24
|
-
spec.files =
|
24
|
+
spec.files = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
|
25
25
|
|
26
26
|
spec.add_dependency "addressable"
|
27
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nethttputils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Maslov aka Nakilon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -40,10 +40,7 @@ executables: []
|
|
40
40
|
extensions: []
|
41
41
|
extra_rdoc_files: []
|
42
42
|
files:
|
43
|
-
- ".travis.yml"
|
44
|
-
- Gemfile
|
45
43
|
- LICENSE
|
46
|
-
- Rakefile
|
47
44
|
- lib/nethttputils.rb
|
48
45
|
- nethttputils.gemspec
|
49
46
|
homepage: https://github.com/nakilon/nethttputils
|
data/.travis.yml
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
---
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
script: "bundle install && bundle exec ruby lib/nethttputils.rb"
|
5
|
-
|
6
|
-
os:
|
7
|
-
- linux
|
8
|
-
- osx
|
9
|
-
rvm:
|
10
|
-
- ruby-head
|
11
|
-
- 2.5
|
12
|
-
- 2.4
|
13
|
-
- 2.3
|
14
|
-
- 2.2
|
15
|
-
- 2.1
|
16
|
-
- 2.0
|
17
|
-
- jruby-head
|
18
|
-
matrix:
|
19
|
-
allow_failures:
|
20
|
-
# something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
|
21
|
-
- rvm: jruby-head
|
22
|
-
- rvm: 2.3
|
23
|
-
os: osx
|
24
|
-
- rvm: 2.2
|
25
|
-
os: osx
|
26
|
-
- rvm: 2.1
|
27
|
-
os: osx
|
28
|
-
- rvm: 2.0
|
29
|
-
os: osx
|
data/Gemfile
DELETED
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "bundler/gem_tasks"
|