nethttputils 0.4.1.0 → 0.4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ff2b86fcc4b55f5f9ea7818eb61b47458da62fa0
4
- data.tar.gz: 610d74457c21418c24b092a65ba47abfd5571b37
3
+ metadata.gz: 6b310f69be9a582878a60741eb385b2cbc61884b
4
+ data.tar.gz: fee129582c51e0ec02abc63e318a681534edc391
5
5
  SHA512:
6
- metadata.gz: 819dc2799c568c4b4aa4d199d14e3f5a0b70253bc0eb6831205482274abb2bbe90f7fb8504d15f6e4561dfa2d1e7d464a4534943a2d6ebd188d7e345752cfc39
7
- data.tar.gz: cde0231d2c7779642ecf68d052cc117178b9905ba8a1bf3427d7c901050f05b75d290624fda7c5a2e2a2f4ed5fc7114390aad00bcdcbc45bab8783d2ee844ff0
6
+ metadata.gz: d7f628d6558dbe9aa0d08125c5a294c957235858829013ac4dd655b0b28fc63e38bed1827ce35fbb9de1d66ca6acde828fa3a5d77ea90a3e08238751cab9b0ea
7
+ data.tar.gz: ea174acf8e166ef034d6d151b171babdb19fa3736cd23fbdb006c460deb28236f1ce33c1bbc72194d30695498ccc1292108063b6319b54eace9631a5b3e68bc7
data/lib/nethttputils.rb CHANGED
@@ -33,7 +33,8 @@ module NetHTTPUtils
33
33
  gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
34
34
  end
35
35
 
36
- def start_http url, max_start_http_retry_delay = 3600, timeout = 30
36
+ def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
37
+ timeout ||= 30
37
38
  uri = url
38
39
  uri = URI.parse begin
39
40
  URI url
@@ -45,6 +46,7 @@ module NetHTTPUtils
45
46
  begin
46
47
  Net::HTTP.start(
47
48
  uri.host, uri.port,
49
+ *(proxy.split ?: if proxy),
48
50
  use_ssl: uri.scheme == "https",
49
51
  verify_mode: OpenSSL::SSL::VERIFY_NONE,
50
52
  **({open_timeout: timeout}), # if timeout
@@ -66,7 +68,7 @@ module NetHTTPUtils
66
68
  end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
67
69
  http
68
70
  end
69
- rescue Errno::ECONNREFUSED => e
71
+ rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
70
72
  if max_start_http_retry_delay < delay *= 2
71
73
  e.message.concat " to #{uri}"
72
74
  raise
@@ -74,10 +76,6 @@ module NetHTTPUtils
74
76
  logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
75
77
  sleep delay
76
78
  retry
77
- rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
78
- logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
79
- sleep 5
80
- retry
81
79
  rescue SocketError => e
82
80
  if max_start_http_retry_delay < delay *= 2
83
81
  e.message.concat " to #{uri}"
@@ -103,14 +101,17 @@ module NetHTTPUtils
103
101
  end
104
102
 
105
103
  private
106
- def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_read_retry_delay: 3600, patch_request: nil, &block
107
- logger = NetHTTPUtils.logger
104
+ def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, force_post: false, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
105
+ timeout ||= 30
106
+ logger = NetHTTPUtils.logger
107
+ logger.info [mtd, http].inspect
108
108
 
109
109
  uri = http.instance_variable_get :@uri
110
- logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query && !form.empty?
110
+ if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty? # not .upcase! because it's not defined for Symbol
111
+ logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query
111
112
  # we can't just merge because URI fails to parse such queries as "/?1"
112
-
113
- uri.query = URI.encode_www_form form if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty?
113
+ uri.query = URI.encode_www_form form
114
+ end
114
115
  cookies = {}
115
116
  prepare_request = lambda do |uri|
116
117
  case mtd.upcase
@@ -129,15 +130,20 @@ module NetHTTPUtils
129
130
  request.basic_auth *auth if auth
130
131
  if (mtd == :POST || mtd == :PATCH) && !form.empty?
131
132
  case type
132
- when :json ; request.body = JSON.dump form
133
- request.content_type = "application/json"
134
- when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
133
+ when :json
134
+ request.body = JSON.dump form
135
+ request.content_type = "application/json"
136
+ when :multipart
137
+ request.set_form form, "multipart/form-data"
138
+ when :form
139
+ if form.any?{ |k, v| v.respond_to? :to_path }
135
140
  request.set_form form, "multipart/form-data"
136
141
  else
137
142
  request.set_form_data form
138
143
  request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
139
144
  end
140
- else ; raise "unknown content-type '#{type}'"
145
+ else
146
+ raise "unknown content-type '#{type}'"
141
147
  end
142
148
  end
143
149
  header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
@@ -172,7 +178,7 @@ module NetHTTPUtils
172
178
  delay = 5
173
179
  response = begin
174
180
  http.request request, &block
175
- rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
181
+ rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
176
182
  raise if max_read_retry_delay < delay *= 2
177
183
  logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
178
184
  sleep delay
@@ -180,12 +186,13 @@ module NetHTTPUtils
180
186
  rescue EOFError => e
181
187
  raise unless e.backtrace.empty?
182
188
  # https://bugs.ruby-lang.org/issues/13018
183
- # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html?spref=reddit
189
+ # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
184
190
  raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
185
191
  end
186
192
  # response.instance_variable_set "@nethttputils_close", http.method(:finish)
187
193
  # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
188
194
 
195
+ now = Time.now
189
196
  remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
190
197
  logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
191
198
  [
@@ -197,7 +204,13 @@ module NetHTTPUtils
197
204
  [
198
205
  response.fetch("x-rate-limit-remaining").to_i,
199
206
  response.fetch("x-rate-limit-reset").to_i,
200
- Time.now.to_i,
207
+ now.to_i,
208
+ ]
209
+ elsif response.key? "x-ratelimit-remaining"
210
+ [
211
+ response.fetch("x-ratelimit-remaining").to_i,
212
+ now + response.fetch("x-ratelimit-reset").to_i,
213
+ now.to_i,
201
214
  ]
202
215
  end
203
216
  if remaining
@@ -222,8 +235,12 @@ module NetHTTPUtils
222
235
  response.add_field "Set-Cookie", "#{k}=#{v}"
223
236
  end
224
237
 
238
+ logger.info "response.code = #{response.code}"
225
239
  case response.code
240
+ when /\A20/
241
+ response
226
242
  when /\A30\d\z/
243
+ next response if no_redirect
227
244
  logger.info "redirect: #{response["location"]}"
228
245
  require "addressable"
229
246
  new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
@@ -234,10 +251,10 @@ module NetHTTPUtils
234
251
  http.use_ssl? != (new_uri.scheme == "https")
235
252
  logger.debug "changing host from '#{http.address}' to '#{new_host}'"
236
253
  # http.finish # why commented out?
237
- http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
254
+ http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
238
255
  end
239
- if request.method == "POST"
240
- logger.info "POST redirects to GET (RFC)"
256
+ if !force_post && request.method == "POST"
257
+ logger.info "POST redirects to GET (RFC)" # TODO: do it only on code 307; note that some servers still do 302
241
258
  mtd = :GET
242
259
  end
243
260
  do_request.call prepare_request[new_uri]
@@ -270,11 +287,10 @@ module NetHTTPUtils
270
287
  end
271
288
  }"
272
289
  response
273
- when /\A20/
274
- response
275
290
  else
276
- logger.warn "code #{response.code} at #{request.method} #{request.uri} from #{
277
- [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
291
+ logger.warn "code #{response.code} from #{request.method} #{request.uri} at #{
292
+ caller_path, caller_locs = caller_locations.chunk(&:path).first
293
+ [caller_path, caller_locs.map(&:lineno).chunk(&:itself).map(&:first)].join ":"
278
294
  }"
279
295
  logger.debug "< body: #{
280
296
  response.body.tap do |body|
@@ -292,11 +308,13 @@ module NetHTTPUtils
292
308
 
293
309
  require "set"
294
310
  @@_405 ||= Set.new
295
- def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
311
+ def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false,
312
+ timeout: nil, no_redirect: false,
296
313
  max_start_http_retry_delay: 3600,
297
314
  max_read_retry_delay: 3600,
298
315
  patch_request: nil, &block
299
- http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
316
+ timeout ||= 30
317
+ http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
300
318
  path = http.instance_variable_get(:@uri).path
301
319
 
302
320
  check_code = lambda do |body|
@@ -327,7 +345,8 @@ module NetHTTPUtils
327
345
  check_code.call body
328
346
  end
329
347
  end
330
- body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
348
+ body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
349
+ timeout: timeout, no_redirect: no_redirect,
331
350
  max_read_retry_delay: max_read_retry_delay,
332
351
  patch_request: patch_request, &block
333
352
  check_code.call body
@@ -337,7 +356,6 @@ module NetHTTPUtils
337
356
  Zlib::GzipReader.new(StringIO.new(body)).read
338
357
  else
339
358
  body
340
- end.tap do |string|
341
359
  end
342
360
  # ensure
343
361
  # response.instance_variable_get("@nethttputils_close").call if response
@@ -479,13 +497,13 @@ if $0 == __FILE__
479
497
  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
480
498
  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
481
499
  [
482
- ["https://imgur.com/a/cccccc"],
483
- ["https://imgur.com/mM4Dh7Z"],
500
+ # ["https://imgur.com/a/oacI3gl"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
501
+ # ["https://imgur.com/mM4Dh7Z"], # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
484
502
  ["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
485
503
  ].each do |url, expectation|
486
504
  begin
487
505
  puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
488
- fail
506
+ fail url
489
507
  rescue NetHTTPUtils::Error => e
490
508
  raise e.code.inspect unless e.code == 404
491
509
  raise e.to_s if e.to_s != expectation if expectation
data/nethttputils.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "nethttputils"
3
- spec.version = "0.4.1.0"
3
+ spec.version = "0.4.2.0"
4
4
  spec.summary = "this tool is like a pet that I adopted young and now I depend on, sorry"
5
5
  spec.description = <<-EOF
6
6
  Back in 2015 I was a guy automating things at my job and two scripts had a common need --
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
21
21
  spec.license = "MIT"
22
22
 
23
23
  spec.require_path = "lib"
24
- spec.files = `git ls-files -z`.split(?\0) - spec.test_files
24
+ spec.files = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
25
25
 
26
26
  spec.add_dependency "addressable"
27
27
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nethttputils
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1.0
4
+ version: 0.4.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Victor Maslov aka Nakilon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-08-23 00:00:00.000000000 Z
11
+ date: 2021-10-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -40,10 +40,7 @@ executables: []
40
40
  extensions: []
41
41
  extra_rdoc_files: []
42
42
  files:
43
- - ".travis.yml"
44
- - Gemfile
45
43
  - LICENSE
46
- - Rakefile
47
44
  - lib/nethttputils.rb
48
45
  - nethttputils.gemspec
49
46
  homepage: https://github.com/nakilon/nethttputils
data/.travis.yml DELETED
@@ -1,29 +0,0 @@
1
- ---
2
- language: ruby
3
-
4
- script: "bundle install && bundle exec ruby lib/nethttputils.rb"
5
-
6
- os:
7
- - linux
8
- - osx
9
- rvm:
10
- - ruby-head
11
- - 2.5
12
- - 2.4
13
- - 2.3
14
- - 2.2
15
- - 2.1
16
- - 2.0
17
- - jruby-head
18
- matrix:
19
- allow_failures:
20
- # something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
21
- - rvm: jruby-head
22
- - rvm: 2.3
23
- os: osx
24
- - rvm: 2.2
25
- os: osx
26
- - rvm: 2.1
27
- os: osx
28
- - rvm: 2.0
29
- os: osx
data/Gemfile DELETED
@@ -1,3 +0,0 @@
1
- source "https://rubygems.org"
2
- gem "byebug"
3
- gemspec
data/Rakefile DELETED
@@ -1 +0,0 @@
1
- require "bundler/gem_tasks"