berkeley_library-util 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a17e4e3e2e6e03e0dda795450e724650a0a7f7ae44db3b7a558bc371d8bed1b7
4
- data.tar.gz: 106ff7e21c1e9568f4bdabc116023c292878f4fd629e5ee3ada934b2c2a10b43
3
+ metadata.gz: 81d92682b9ea2f5466198ab26b0a31775704f3d52408e67453b1af8871367aa8
4
+ data.tar.gz: 296f6fd7ad69c125e732b423f9eb75f338cc866b9a7c19dae6c550d754a2c285
5
5
  SHA512:
6
- metadata.gz: b4fbde8aa9e43a7ad7c414b92b488c5d2af1ae29c7f28a7ecd8f6ceb308e42c3c27a2256994362b15cba5133ec00feb32d5fe76f0b774577b292be6c2ba08528
7
- data.tar.gz: 23c41cbed1b0befc458c8989792c45cf7429905b06e2deec72a88478f8eb7676a6e3be96dae5fa308239bb4f70b19acae421593711c20dd22d305f55262b0f64
6
+ metadata.gz: f632ee12e255ca16fcd22effc3df7d1ddcf7cb4a93ac49c4ca7d3beef8ba29143b8bc1f96097236fdf9dde1802625423990052ac12a4920cb0e8a074715b3cf4
7
+ data.tar.gz: c9ab1aa8a055caf6ba968af662fdcf3dbe782fdce4c0e884032db10a427dae4049f48fde9297d415336374760183ec23058c355bed3ee8bb9f90593bf3844841
data/CHANGES.md CHANGED
@@ -1,3 +1,14 @@
1
+ # 0.1.8 (2023-03-20)
2
+
3
+ - Add `Retry-After` support to `Requester` for `429 Too Many Requests` and `503 Service Unavailable`.
4
+
5
+ # 0.1.7 (2023-03-15)
6
+
7
+ - Allow passing `log: false` to `Requester` methods (and corresponding `URIs` convenience
8
+ methods) to suppress logging of each request URL and response code.
9
+ - Allow constructing `Requester` instances with delayed execution
10
+ - Fix documentation for `get_response` and `head_response` in `URIs` and `Requester`
11
+
1
12
  # 0.1.6 (2023-03-09)
2
13
 
3
14
  - Fix issue in `Requester` where query parameters would not be appended properly
@@ -7,7 +7,7 @@ module BerkeleyLibrary
7
7
  SUMMARY = 'Miscellaneous Ruby utilities for the UC Berkeley Library'.freeze
8
8
  DESCRIPTION = 'A collection of miscellaneous Ruby routines for the UC Berkeley Library.'.freeze
9
9
  LICENSE = 'MIT'.freeze
10
- VERSION = '0.1.6'.freeze
10
+ VERSION = '0.1.8'.freeze
11
11
  HOMEPAGE = 'https://github.com/BerkeleyLibrary/util'.freeze
12
12
  end
13
13
  end
@@ -39,6 +39,14 @@ module BerkeleyLibrary
39
39
  end
40
40
  end
41
41
 
42
+ # Returns the new URI as a string.
43
+ #
44
+ # @return [String] a new URI appending the joined path elements, as a string.
45
+ # @raise URI::InvalidComponentError if appending the specified elements would create an invalid URI
46
+ def to_url_str
47
+ to_uri.to_s
48
+ end
49
+
42
50
  private
43
51
 
44
52
  def handle_element(element, elem_index)
@@ -0,0 +1,15 @@
1
+ class RetryDelayTooLarge < RestClient::Exception
2
+ def initialize(response, delay:, max_delay:)
3
+ super(response, response.code)
4
+
5
+ self.message = 'Retry delay of %0.2gs exceeds limit of %0.2gs' % [delay, max_delay]
6
+ end
7
+ end
8
+
9
+ class RetryLimitExceeded < RestClient::Exception
10
+ def initialize(response, max_retries:)
11
+ super(response, response.code)
12
+
13
+ self.message = "Retry limit (#{max_retries}) exceeded"
14
+ end
15
+ end
@@ -0,0 +1,89 @@
1
+ module BerkeleyLibrary
2
+ module Util
3
+ module URIs
4
+ class Requester
5
+ # rubocop:disable Metrics/ParameterLists
6
+ module ClassMethods
7
+ # Performs a GET request and returns the response body as a string.
8
+ #
9
+ # @param uri [URI, String] the URI to GET
10
+ # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
11
+ # @param headers [Hash] the request headers.
12
+ # @return [String] the body as a string.
13
+ # @param log [Boolean] whether to log each request URL and response code
14
+ # @param max_retries [Integer] the maximum number of times to retry after a 429 or 503 with Retry-After
15
+ # @param max_retry_delay [Integer] the maximum retry delay (in seconds) to accept in a Retry-After header
16
+ # @raise [RestClient::Exception] in the event of an unsuccessful request.
17
+ def get(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
18
+ resp = make_request(:get, uri, params, headers, log, max_retries, max_retry_delay)
19
+ resp.body
20
+ end
21
+
22
+ # Performs a HEAD request and returns the response status as an integer.
23
+ # Note that unlike {Requester#get}, this does not raise an error in the
24
+ # event of an unsuccessful request.
25
+ #
26
+ # @param uri [URI, String] the URI to HEAD
27
+ # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
28
+ # @param headers [Hash] the request headers.
29
+ # @param log [Boolean] whether to log each request URL and response code
30
+ # @return [Integer] the response code as an integer.
31
+ def head(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
32
+ head_response(uri, params: params, headers: headers, log: log, max_retries: max_retries, max_retry_delay: max_retry_delay).code
33
+ end
34
+
35
+ # Performs a GET request and returns the response, even in the event of
36
+ # a failed request.
37
+ #
38
+ # @param uri [URI, String] the URI to GET
39
+ # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
40
+ # @param headers [Hash] the request headers.
41
+ # @param log [Boolean] whether to log each request URL and response code
42
+ # @return [RestClient::Response] the response
43
+ def get_response(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
44
+ make_request(:get, uri, params, headers, log, max_retries, max_retry_delay)
45
+ rescue RestClient::Exception => e
46
+ e.response
47
+ end
48
+
49
+ # Performs a HEAD request and returns the response, even in the event of
50
+ # a failed request.
51
+ #
52
+ # @param uri [URI, String] the URI to HEAD
53
+ # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
54
+ # @param headers [Hash] the request headers.
55
+ # @param log [Boolean] whether to log each request URL and response code
56
+ # @return [RestClient::Response] the response
57
+ def head_response(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
58
+ make_request(:head, uri, params, headers, log, max_retries, max_retry_delay)
59
+ rescue RestClient::Exception => e
60
+ e.response
61
+ end
62
+
63
+ private
64
+
65
+ def make_request(method, url, params, headers, log, max_retries, max_retry_delay)
66
+ Requester.new(
67
+ method,
68
+ url,
69
+ params: params,
70
+ headers: headers,
71
+ log: log,
72
+ max_retries: max_retries,
73
+ max_retry_delay: max_retry_delay
74
+ ).make_request
75
+ end
76
+
77
+ end
78
+ # rubocop:enable Metrics/ParameterLists
79
+
80
+ # ------------------------------------------------------------
81
+ # Class methods
82
+
83
+ class << self
84
+ include ClassMethods
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
@@ -1,108 +1,166 @@
1
+ require 'time'
1
2
  require 'rest-client'
2
3
  require 'berkeley_library/util/uris/appender'
4
+ require 'berkeley_library/util/uris/exceptions'
3
5
  require 'berkeley_library/util/uris/validator'
6
+ require 'berkeley_library/util/uris/requester/class_methods'
4
7
  require 'berkeley_library/logging'
5
8
 
6
9
  module BerkeleyLibrary
7
10
  module Util
8
11
  module URIs
9
- module Requester
10
- class << self
11
- include BerkeleyLibrary::Logging
12
-
13
- # Performs a GET request and returns the response body as a string.
14
- #
15
- # @param uri [URI, String] the URI to GET
16
- # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
17
- # @param headers [Hash] the request headers.
18
- # @return [String] the body as a string.
19
- # @raise [RestClient::Exception] in the event of an unsuccessful request.
20
- def get(uri, params: {}, headers: {})
21
- resp = make_request(:get, uri, params, headers)
22
- resp.body
23
- end
12
+ class Requester
13
+ include BerkeleyLibrary::Logging
24
14
 
25
- # Performs a HEAD request and returns the response status as an integer.
26
- # Note that unlike {Requester#get}, this does not raise an error in the
27
- # event of an unsuccessful request.
28
- #
29
- # @param uri [URI, String] the URI to HEAD
30
- # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
31
- # @param headers [Hash] the request headers.
32
- # @return [Integer] the response code as an integer.
33
- def head(uri, params: {}, headers: {})
34
- head_response(uri, params: params, headers: headers).code
35
- end
15
+ # ------------------------------------------------------------
16
+ # Constants
36
17
 
37
- # Performs a GET request and returns the response, even in the event of
38
- # a failed request.
39
- #
40
- # @param uri [URI, String] the URI to GET
41
- # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
42
- # @param headers [Hash] the request headers.
43
- # @return [RestClient::Response] the body as a string.
44
- def get_response(uri, params: {}, headers: {})
45
- make_request(:get, uri, params, headers)
46
- rescue RestClient::Exception => e
47
- e.response
48
- end
18
+ SUPPORTED_METHODS = %i[get head].freeze
19
+ RETRY_HEADER = :retry_after
20
+ RETRY_STATUSES = [429, 503].freeze
21
+ MAX_RETRY_DELAY_SECONDS = 10
22
+ MAX_RETRIES = 3
49
23
 
50
- # Performs a HEAD request and returns the response, even in the event of
51
- # a failed request.
52
- #
53
- # @param uri [URI, String] the URI to HEAD
54
- # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
55
- # @param headers [Hash] the request headers.
56
- # @return [RestClient::Response] the body as a string.
57
- def head_response(uri, params: {}, headers: {})
58
- make_request(:head, uri, params, headers)
59
- rescue RestClient::Exception => e
60
- e.response
61
- end
24
+ # ------------------------------------------------------------
25
+ # Attributes
26
+
27
+ attr_reader :method, :url_str, :headers, :log, :max_retries, :max_retry_delay
28
+
29
+ # ------------------------------------------------------------
30
+ # Initializer
31
+
32
+ # Initializes a new Requester.
33
+ #
34
+ # @param method [:get, :head] the HTTP method to use
35
+ # @param url [String, URI] the URL or URI to request
36
+ # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
37
+ # @param headers [Hash] the request headers.
38
+ # @param log [Boolean] whether to log each request URL and response code
39
+ # @param max_retries [Integer] the maximum number of times to retry after a 429 or 503 with Retry-After
40
+ # @param max_retry_delay [Integer] the maximum retry delay (in seconds) to accept in a Retry-After header
41
+ # @raise URI::InvalidURIError if the specified URL is invalid
42
+ # rubocop:disable Metrics/ParameterLists
43
+ def initialize(method, url, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
44
+ raise ArgumentError, "#{method} not supported" unless SUPPORTED_METHODS.include?(method)
45
+ raise ArgumentError, 'url cannot be nil' unless (uri = Validator.uri_or_nil(url))
46
+
47
+ @method = method
48
+ @url_str = url_str_with_params(uri, params)
49
+ @headers = headers
50
+ @log = log
51
+ @max_retries = max_retries
52
+ @max_retry_delay = max_retry_delay
53
+ end
54
+
55
+ # rubocop:enable Metrics/ParameterLists
62
56
 
63
- private
57
+ # ------------------------------------------------------------
58
+ # Public instance methods
64
59
 
65
- # @return [RestClient::Response]
66
- def make_request(method, uri, params, headers)
67
- url_str = url_str_with_params(uri, params)
68
- req_resp_or_raise(method, url_str, headers)
60
+ # @return [RestClient::Response]
61
+ def make_request
62
+ execute_request.tap do |resp|
63
+ log_response(resp)
69
64
  end
65
+ rescue RestClient::Exception => e
66
+ log_response(e.response)
67
+ raise
68
+ end
69
+
70
+ # ------------------------------------------------------------
71
+ # Private methods
70
72
 
71
- def url_str_with_params(url, params)
72
- raise ArgumentError, 'url cannot be nil' unless (uri = Validator.uri_or_nil(url))
73
+ private
73
74
 
74
- elements = [].tap do |ee|
75
- ee << uri
76
- next if params.empty?
75
+ def log_response(response)
76
+ return unless log
77
77
 
78
- ee << (uri.query ? '&' : '?')
79
- ee << URI.encode_www_form(params)
80
- end
78
+ logger.info("#{method.to_s.upcase} #{url_str} returned #{response.code}")
79
+ end
81
80
 
82
- uri = Appender.new(*elements).to_uri
83
- uri.to_s
81
+ def url_str_with_params(uri, params)
82
+ elements = [uri]
83
+ if params.any?
84
+ elements << (uri.query ? '&' : '?')
85
+ elements << URI.encode_www_form(params)
84
86
  end
85
87
 
86
- # @return [RestClient::Response]
87
- def req_resp_or_raise(method, url_str, headers)
88
- resp = RestClient::Request.execute(method: method, url: url_str, headers: headers)
89
- begin
90
- return resp if (status = resp.code) == 200
91
-
92
- raise(exception_for(resp, status))
93
- ensure
94
- # noinspection RubyMismatchedReturnType
95
- logger.info("#{method.to_s.upcase} #{url_str} returned #{status}")
96
- end
88
+ Appender.new(*elements).to_url_str
89
+ end
90
+
91
+ def execute_request(retries_remaining = max_retries)
92
+ try_execute_request
93
+ rescue RestClient::Exception => e
94
+ response = e.response
95
+ raise unless (retry_delay = retry_delay_from(response))
96
+
97
+ wait_for_retry(response, retry_delay, retries_remaining)
98
+ execute_request(retries_remaining - 1)
99
+ end
100
+
101
+ def try_execute_request
102
+ RestClient::Request.execute(method: method, url: url_str, headers: headers).tap do |response|
103
+ # Not all failed RestClient requests throw exceptions
104
+ raise(exception_for(response)) unless response.code == 200
97
105
  end
106
+ end
107
+
108
+ def wait_for_retry(response, retry_delay, retries_remaining)
109
+ raise RetryLimitExceeded.new(response, max_retries: max_retries) unless retries_remaining > 0
110
+ raise RetryDelayTooLarge.new(response, delay: retry_delay, max_delay: max_retry_delay) if retry_delay > max_retry_delay
98
111
 
99
- def exception_for(resp, status)
100
- RestClient::RequestFailed.new(resp, status).tap do |ex|
101
- status_message = RestClient::STATUSES[status] || '(Unknown)'
102
- ex.message = "#{status} #{status_message}"
103
- end
112
+ sleep(retry_delay)
113
+ end
114
+
115
+ def exception_for(resp)
116
+ status = resp.code
117
+ ex_class_for(status).new(resp, status).tap do |ex|
118
+ status_message = RestClient::STATUSES[status] || '(Unknown)'
119
+ ex.message = "#{status} #{status_message}"
104
120
  end
105
121
  end
122
+
123
+ def ex_class_for(status)
124
+ RestClient::Exceptions::EXCEPTIONS_MAP[status] || RestClient::RequestFailed
125
+ end
126
+
127
+ # Returns the retry interval for the specified exception, or `nil`
128
+ # if the response does not allow a retry.
129
+ #
130
+ # @param resp [RestClient::Response] the response
131
+ # @return [Integer, nil] the retry delay in seconds, or `nil` if the response
132
+ # does not allow a retry
133
+ def retry_delay_from(resp)
134
+ return unless RETRY_STATUSES.include?(resp.code)
135
+ return unless (retry_header_value = resp.headers[RETRY_HEADER])
136
+ return unless (retry_delay_seconds = parse_retry_header_value(retry_header_value))
137
+
138
+ [1, retry_delay_seconds.ceil].max
139
+ end
140
+
141
+ # @return [Float, nil] the retry delay in seconds, or `nil` if the delay cannot be parsed
142
+ def parse_retry_header_value(v)
143
+ # start by assuming it's a delay in seconds
144
+ Float(v) # should be an integer but let's not count on it
145
+ rescue ArgumentError
146
+ # assume it's an HTTP-date
147
+ parse_retry_after_date(v)
148
+ end
149
+
150
+ # Parses the specified RFC2822 datetime string and returns the interval between that
151
+ # datetime and the current time in seconds
152
+ #
153
+ # @param date_str [String] an RFC2822 datetime string
154
+ # @return [Float, nil] the interval between the current time and the specified datetime,
155
+ # or nil if `date_str` cannot be parsed
156
+ def parse_retry_after_date(date_str)
157
+ retry_after = DateTime.rfc2822(date_str).to_time
158
+ retry_after - Time.now
159
+ rescue ArgumentError
160
+ logger.warn("Can't parse #{RETRY_HEADER} value #{date_str}")
161
+ nil
162
+ end
163
+
106
164
  end
107
165
  end
108
166
  end
@@ -31,10 +31,11 @@ module BerkeleyLibrary
31
31
  # @param uri [URI, String] the URI to GET
32
32
  # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
33
33
  # @param headers [Hash] the request headers.
34
+ # @param log [Boolean] whether to log each request URL and response code
34
35
  # @return [String] the body as a string.
35
36
  # @raise [RestClient::Exception] in the event of an unsuccessful request.
36
- def get(uri, params: {}, headers: {})
37
- Requester.get(uri, params: params, headers: headers)
37
+ def get(uri, params: {}, headers: {}, log: true)
38
+ Requester.get(uri, params: params, headers: headers, log: log)
38
39
  end
39
40
 
40
41
  # Performs a HEAD request and returns the response status as an integer.
@@ -44,9 +45,10 @@ module BerkeleyLibrary
44
45
  # @param uri [URI, String] the URI to HEAD
45
46
  # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
46
47
  # @param headers [Hash] the request headers.
48
+ # @param log [Boolean] whether to log each request URL and response code
47
49
  # @return [Integer] the response code as an integer.
48
- def head(uri, params: {}, headers: {})
49
- Requester.head(uri, params: params, headers: headers)
50
+ def head(uri, params: {}, headers: {}, log: true)
51
+ Requester.head(uri, params: params, headers: headers, log: log)
50
52
  end
51
53
 
52
54
  # Performs a GET request and returns the response, even in the event of
@@ -55,9 +57,10 @@ module BerkeleyLibrary
55
57
  # @param uri [URI, String] the URI to GET
56
58
  # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
57
59
  # @param headers [Hash] the request headers.
58
- # @return [RestClient::Response] the body as a string.
59
- def get_response(uri, params: {}, headers: {})
60
- Requester.get_response(uri, params: params, headers: headers)
60
+ # @param log [Boolean] whether to log each request URL and response code
61
+ # @return [RestClient::Response] the response
62
+ def get_response(uri, params: {}, headers: {}, log: true)
63
+ Requester.get_response(uri, params: params, headers: headers, log: log)
61
64
  end
62
65
 
63
66
  # Performs a HEAD request and returns the response, even in the event of
@@ -66,9 +69,10 @@ module BerkeleyLibrary
66
69
  # @param uri [URI, String] the URI to HEAD
67
70
  # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
68
71
  # @param headers [Hash] the request headers.
69
- # @return [RestClient::Response] the body as a string.
70
- def head_response(uri, params: {}, headers: {})
71
- Requester.head_response(uri, params: params, headers: headers)
72
+ # @param log [Boolean] whether to log each request URL and response code
73
+ # @return [RestClient::Response] the response
74
+ def head_response(uri, params: {}, headers: {}, log: true)
75
+ Requester.head_response(uri, params: params, headers: headers, log: log)
72
76
  end
73
77
 
74
78
  # Returns the specified URL as a URI, or `nil` if the URL is `nil`.
@@ -65,6 +65,250 @@ module BerkeleyLibrary
65
65
  result = Requester.get(url1)
66
66
  expect(result).to eq(expected_body)
67
67
  end
68
+
69
+ describe 'retries' do
70
+ let(:url) { 'https://example.org/' }
71
+ let(:expected_body) { 'Help! I am trapped in a unit test' }
72
+
73
+ context 'handling 429 Too Many Requests' do
74
+ context 'with Retry-After' do
75
+ context 'in seconds' do
76
+ it 'retries after the specified delay' do
77
+ retry_after_seconds = 1
78
+
79
+ stub_request(:get, url)
80
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
81
+ .to_return(status: 200, body: expected_body)
82
+
83
+ requester = Requester.new(:get, url)
84
+ expect(requester).to receive(:sleep).with(1).once
85
+
86
+ result = requester.make_request
87
+ expect(result).to eq(expected_body)
88
+ end
89
+
90
+ it 'handles a non-integer retry delay' do
91
+ retry_after_seconds = 1.5
92
+ stub_request(:get, url)
93
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
94
+ .to_return(status: 200, body: expected_body)
95
+
96
+ requester = Requester.new(:get, url)
97
+ expect(requester).to receive(:sleep).with(2).once
98
+
99
+ result = requester.make_request
100
+ expect(result).to eq(expected_body)
101
+ end
102
+
103
+ it 'raises RetryDelayTooLarge if the delay is too large' do
104
+ retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
105
+
106
+ stub_request(:get, url)
107
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
108
+
109
+ requester = Requester.new(:get, url)
110
+ expect(requester).not_to receive(:sleep)
111
+
112
+ expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
113
+ expect(ex.cause).to be_a(RestClient::TooManyRequests)
114
+ end
115
+ end
116
+
117
+ it 'raises RetryLimitExceeded if there are too many retries' do
118
+ retry_after_seconds = 1
119
+
120
+ stub_request(:get, url)
121
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
122
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
123
+
124
+ requester = Requester.new(:get, url, max_retries: 1)
125
+ expect(requester).to receive(:sleep).with(1).once
126
+
127
+ expect { requester.make_request }.to raise_error(RetryLimitExceeded) do |ex|
128
+ expect(ex.cause).to be_a(RestClient::TooManyRequests)
129
+ end
130
+ end
131
+ end
132
+
133
+ context 'as RFC2822 datetime' do
134
+ it 'retries after the specified delay' do
135
+ retry_after_seconds = 1
136
+ retry_after_datetime = (Time.now + retry_after_seconds)
137
+
138
+ stub_request(:get, url)
139
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
140
+ .to_return(status: 200, body: expected_body)
141
+
142
+ requester = Requester.new(:get, url)
143
+ expect(requester).to receive(:sleep).with(1).once
144
+
145
+ result = requester.make_request
146
+ expect(result).to eq(expected_body)
147
+ end
148
+
149
+ it 'handles a non-integer retry delay' do
150
+ retry_after_seconds = 2.75
151
+ retry_after_datetime = (Time.now + retry_after_seconds)
152
+
153
+ stub_request(:get, url)
154
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
155
+ .to_return(status: 200, body: expected_body)
156
+
157
+ requester = Requester.new(:get, url)
158
+ expected_value = a_value_within(1).of(retry_after_seconds)
159
+ expect(requester).to receive(:sleep).with(expected_value).once
160
+
161
+ result = requester.make_request
162
+ expect(result).to eq(expected_body)
163
+ end
164
+
165
+ it 'raises RetryDelayTooLarge if the delay is too large' do
166
+ retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
167
+ retry_after_datetime = (Time.now + retry_after_seconds)
168
+
169
+ stub_request(:get, url)
170
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
171
+
172
+ requester = Requester.new(:get, url)
173
+ expect(requester).not_to receive(:sleep)
174
+
175
+ expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
176
+ expect(ex.cause).to be_a(RestClient::TooManyRequests)
177
+ end
178
+ end
179
+
180
+ it 'raises RetryLimitExceeded if there are too many retries' do
181
+ retry_after_seconds = 1
182
+ retry_after_datetime = (Time.now + retry_after_seconds)
183
+
184
+ stub_request(:get, url)
185
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
186
+ .to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
187
+
188
+ requester = Requester.new(:get, url, max_retries: 1)
189
+ expect(requester).to receive(:sleep).with(1).once
190
+
191
+ expect { requester.make_request }.to raise_error(RetryLimitExceeded) do |ex|
192
+ expect(ex.cause).to be_a(RestClient::TooManyRequests)
193
+ end
194
+ end
195
+
196
+ end
197
+
198
+ it 'ignores an invalid Retry-After' do
199
+ stub_request(:get, url)
200
+ .to_return(status: 429, headers: { 'Retry-After' => 'the end of the world' })
201
+
202
+ requester = Requester.new(:get, url)
203
+ expect { requester.make_request }.to raise_error(RestClient::TooManyRequests)
204
+ end
205
+ end
206
+ end
207
+
208
+ context 'handling 503 Service Unavailable' do
209
+ context 'with Retry-After' do
210
+ context 'in seconds' do
211
+ it 'retries after the specified delay' do
212
+ retry_after_seconds = 1
213
+
214
+ stub_request(:get, url)
215
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
216
+ .to_return(status: 200, body: expected_body)
217
+
218
+ requester = Requester.new(:get, url)
219
+ expect(requester).to receive(:sleep).with(1).once
220
+
221
+ result = requester.make_request
222
+ expect(result).to eq(expected_body)
223
+ end
224
+
225
+ it 'handles a non-integer retry delay' do
226
+ retry_after_seconds = 0.75
227
+ stub_request(:get, url)
228
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
229
+ .to_return(status: 200, body: expected_body)
230
+
231
+ requester = Requester.new(:get, url)
232
+ expect(requester).to receive(:sleep).with(1).once
233
+
234
+ result = requester.make_request
235
+ expect(result).to eq(expected_body)
236
+ end
237
+
238
+ it 'raises RetryDelayTooLarge if the delay is too large' do
239
+ retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
240
+
241
+ stub_request(:get, url)
242
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
243
+
244
+ requester = Requester.new(:get, url)
245
+ expect(requester).not_to receive(:sleep)
246
+
247
+ expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
248
+ expect(ex.cause).to be_a(RestClient::ServiceUnavailable)
249
+ end
250
+ end
251
+ end
252
+
253
+ context 'as RFC2822 datetime' do
254
+ it 'retries after the specified delay' do
255
+ retry_after_seconds = 1
256
+ retry_after_datetime = (Time.now + retry_after_seconds)
257
+
258
+ stub_request(:get, url)
259
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
260
+ .to_return(status: 200, body: expected_body)
261
+
262
+ requester = Requester.new(:get, url)
263
+ expect(requester).to receive(:sleep).with(1).once
264
+
265
+ result = requester.make_request
266
+ expect(result).to eq(expected_body)
267
+ end
268
+
269
+ it 'handles a non-integer retry delay' do
270
+ retry_after_seconds = 1.75
271
+ retry_after_datetime = (Time.now + retry_after_seconds)
272
+
273
+ stub_request(:get, url)
274
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
275
+ .to_return(status: 200, body: expected_body)
276
+
277
+ requester = Requester.new(:get, url)
278
+ expected_value = a_value_within(1).of(retry_after_seconds)
279
+ expect(requester).to receive(:sleep).with(expected_value).once
280
+
281
+ result = requester.make_request
282
+ expect(result).to eq(expected_body)
283
+ end
284
+
285
+ it 'raises RetryDelayTooLarge if the delay is too large' do
286
+ retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
287
+ retry_after_datetime = (Time.now + retry_after_seconds)
288
+
289
+ stub_request(:get, url)
290
+ .to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
291
+
292
+ requester = Requester.new(:get, url)
293
+ expect(requester).not_to receive(:sleep)
294
+
295
+ expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
296
+ expect(ex.cause).to be_a(RestClient::ServiceUnavailable)
297
+ end
298
+ end
299
+ end
300
+
301
+ it 'ignores an invalid Retry-After' do
302
+ stub_request(:get, url)
303
+ .to_return(status: 503, headers: { 'Retry-After' => 'the end of the world' })
304
+
305
+ requester = Requester.new(:get, url)
306
+ expect { requester.make_request }.to raise_error(RestClient::ServiceUnavailable)
307
+ end
308
+ end
309
+ end
310
+ end
311
+
68
312
  end
69
313
 
70
314
  describe :head do
@@ -145,6 +389,112 @@ module BerkeleyLibrary
145
389
  expect { Requester.head(nil) }.to raise_error(ArgumentError)
146
390
  end
147
391
  end
392
+
393
+ describe 'logging' do
394
+ attr_reader :logger
395
+
396
+ before do
397
+ @logger = instance_double(BerkeleyLibrary::Logging::Logger)
398
+ allow(BerkeleyLibrary::Logging).to receive(:logger).and_return(logger)
399
+ end
400
+
401
+ context 'GET' do
402
+ it 'logs request URLs and response codes for successful GET requests' do
403
+ url = 'https://example.org/'
404
+ expected_body = 'Help! I am trapped in a unit test'
405
+ stub_request(:get, url).to_return(body: expected_body)
406
+
407
+ expect(logger).to receive(:info).with(/#{url}.*200/)
408
+ Requester.send(:get, url)
409
+ end
410
+
411
+ it 'can suppress logging for successful GET requests' do
412
+ url = 'https://example.org/'
413
+ expected_body = 'Help! I am trapped in a unit test'
414
+ stub_request(:get, url).to_return(body: expected_body)
415
+
416
+ expect(logger).not_to receive(:info)
417
+ Requester.send(:get, url, log: false)
418
+ end
419
+
420
+ it 'logs request URLs and response codes for failed GET requests' do
421
+ url = 'https://example.org/'
422
+ status = 500
423
+ stub_request(:get, url).to_return(status: status)
424
+
425
+ expect(logger).to receive(:info).with(/#{url}.*#{status}/)
426
+ expect { Requester.send(:get, url) }.to raise_error(RestClient::InternalServerError)
427
+ end
428
+
429
+ it 'can suppress logging for failed GET requests' do
430
+ url = 'https://example.org/'
431
+ stub_request(:get, url).to_return(status: 500)
432
+
433
+ expect(logger).not_to receive(:info)
434
+ expect { Requester.send(:get, url, log: false) }.to raise_error(RestClient::InternalServerError)
435
+ end
436
+ end
437
+
438
+ context 'HEAD' do
439
+ it 'logs request URLs and response codes for successful HEAD requests' do
440
+ url = 'https://example.org/'
441
+ expected_body = 'Help! I am trapped in a unit test'
442
+ stub_request(:head, url).to_return(body: expected_body)
443
+
444
+ expect(logger).to receive(:info).with(/#{url}.*200/)
445
+ Requester.send(:head, url)
446
+ end
447
+
448
+ it 'can suppress logging for successful HEAD requests' do
449
+ url = 'https://example.org/'
450
+ expected_body = 'Help! I am trapped in a unit test'
451
+ stub_request(:head, url).to_return(body: expected_body)
452
+
453
+ expect(logger).not_to receive(:info)
454
+ Requester.send(:head, url, log: false)
455
+ end
456
+
457
+ it 'logs request URLs and response codes for failed HEAD requests' do
458
+ url = 'https://example.org/'
459
+ status = 500
460
+ stub_request(:head, url).to_return(status: status)
461
+
462
+ expect(logger).to receive(:info).with(/#{url}.*#{status}/)
463
+ expect(Requester.send(:head, url)).to eq(status)
464
+ end
465
+
466
+ it 'can suppress logging for failed HEAD requests' do
467
+ url = 'https://example.org/'
468
+ status = 500
469
+ stub_request(:head, url).to_return(status: status)
470
+
471
+ expect(logger).not_to receive(:info)
472
+ expect(Requester.send(:head, url, log: false)).to eq(status)
473
+ end
474
+ end
475
+ end
476
+
477
+ describe :new do
478
+ it 'rejects invalid URIs' do
479
+ url = 'not a uri'
480
+ Requester::SUPPORTED_METHODS.each do |method|
481
+ expect { Requester.new(method, url) }.to raise_error(URI::InvalidURIError)
482
+ end
483
+ end
484
+
485
+ it 'rejects nil URIs' do
486
+ Requester::SUPPORTED_METHODS.each do |method|
487
+ expect { Requester.new(method, nil) }.to raise_error(ArgumentError)
488
+ end
489
+ end
490
+
491
+ it 'rejects unsupported methods' do
492
+ url = 'https://example.org/'
493
+ %i[put patch post].each do |method|
494
+ expect { Requester.new(method, url) }.to raise_error(ArgumentError)
495
+ end
496
+ end
497
+ end
148
498
  end
149
499
  end
150
500
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: berkeley_library-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Moles
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-03-09 00:00:00.000000000 Z
11
+ date: 2023-03-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: berkeley_library-logging
@@ -241,7 +241,9 @@ files:
241
241
  - lib/berkeley_library/util/times.rb
242
242
  - lib/berkeley_library/util/uris.rb
243
243
  - lib/berkeley_library/util/uris/appender.rb
244
+ - lib/berkeley_library/util/uris/exceptions.rb
244
245
  - lib/berkeley_library/util/uris/requester.rb
246
+ - lib/berkeley_library/util/uris/requester/class_methods.rb
245
247
  - lib/berkeley_library/util/uris/validator.rb
246
248
  - rakelib/.rubocop.yml
247
249
  - rakelib/coverage.rake