berkeley_library-util 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +11 -0
- data/lib/berkeley_library/util/module_info.rb +1 -1
- data/lib/berkeley_library/util/uris/appender.rb +8 -0
- data/lib/berkeley_library/util/uris/exceptions.rb +15 -0
- data/lib/berkeley_library/util/uris/requester/class_methods.rb +89 -0
- data/lib/berkeley_library/util/uris/requester.rb +139 -81
- data/lib/berkeley_library/util/uris.rb +14 -10
- data/spec/berkeley_library/util/uris/requester_spec.rb +350 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 81d92682b9ea2f5466198ab26b0a31775704f3d52408e67453b1af8871367aa8
|
4
|
+
data.tar.gz: 296f6fd7ad69c125e732b423f9eb75f338cc866b9a7c19dae6c550d754a2c285
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f632ee12e255ca16fcd22effc3df7d1ddcf7cb4a93ac49c4ca7d3beef8ba29143b8bc1f96097236fdf9dde1802625423990052ac12a4920cb0e8a074715b3cf4
|
7
|
+
data.tar.gz: c9ab1aa8a055caf6ba968af662fdcf3dbe782fdce4c0e884032db10a427dae4049f48fde9297d415336374760183ec23058c355bed3ee8bb9f90593bf3844841
|
data/CHANGES.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
# 0.1.8 (2023-03-20)
|
2
|
+
|
3
|
+
- Add `Retry-After` support to `Requester` for `429 Too Many Requests` and `503 Service Unavailable`.
|
4
|
+
|
5
|
+
# 0.1.7 (2023-03-15)
|
6
|
+
|
7
|
+
- Allow passing `log: false` to `Requester` methods (and corresponding `URIs` convenience
|
8
|
+
methods) to suppress logging of each request URL and response code.
|
9
|
+
- Allow constructing `Requester` instances with delayed execution
|
10
|
+
- Fix documentation for `get_response` and `head_response` in `URIs` and `Requester`
|
11
|
+
|
1
12
|
# 0.1.6 (2023-03-09)
|
2
13
|
|
3
14
|
- Fix issue in `Requester` where query parameters would not be appended properly
|
@@ -7,7 +7,7 @@ module BerkeleyLibrary
|
|
7
7
|
SUMMARY = 'Miscellaneous Ruby utilities for the UC Berkeley Library'.freeze
|
8
8
|
DESCRIPTION = 'A collection of miscellaneous Ruby routines for the UC Berkeley Library.'.freeze
|
9
9
|
LICENSE = 'MIT'.freeze
|
10
|
-
VERSION = '0.1.
|
10
|
+
VERSION = '0.1.8'.freeze
|
11
11
|
HOMEPAGE = 'https://github.com/BerkeleyLibrary/util'.freeze
|
12
12
|
end
|
13
13
|
end
|
@@ -39,6 +39,14 @@ module BerkeleyLibrary
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
+
# Returns the new URI as a string.
|
43
|
+
#
|
44
|
+
# @return [String] a new URI appending the joined path elements, as a string.
|
45
|
+
# @raise URI::InvalidComponentError if appending the specified elements would create an invalid URI
|
46
|
+
def to_url_str
|
47
|
+
to_uri.to_s
|
48
|
+
end
|
49
|
+
|
42
50
|
private
|
43
51
|
|
44
52
|
def handle_element(element, elem_index)
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class RetryDelayTooLarge < RestClient::Exception
|
2
|
+
def initialize(response, delay:, max_delay:)
|
3
|
+
super(response, response.code)
|
4
|
+
|
5
|
+
self.message = 'Retry delay of %0.2gs exceeds limit of %0.2gs' % [delay, max_delay]
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
class RetryLimitExceeded < RestClient::Exception
|
10
|
+
def initialize(response, max_retries:)
|
11
|
+
super(response, response.code)
|
12
|
+
|
13
|
+
self.message = "Retry limit (#{max_retries}) exceeded"
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module BerkeleyLibrary
|
2
|
+
module Util
|
3
|
+
module URIs
|
4
|
+
class Requester
|
5
|
+
# rubocop:disable Metrics/ParameterLists
|
6
|
+
module ClassMethods
|
7
|
+
# Performs a GET request and returns the response body as a string.
|
8
|
+
#
|
9
|
+
# @param uri [URI, String] the URI to GET
|
10
|
+
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
11
|
+
# @param headers [Hash] the request headers.
|
12
|
+
# @return [String] the body as a string.
|
13
|
+
# @param log [Boolean] whether to log each request URL and response code
|
14
|
+
# @param max_retries [Integer] the maximum number of times to retry after a 429 or 503 with Retry-After
|
15
|
+
# @param max_retry_delay [Integer] the maximum retry delay (in seconds) to accept in a Retry-After header
|
16
|
+
# @raise [RestClient::Exception] in the event of an unsuccessful request.
|
17
|
+
def get(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
|
18
|
+
resp = make_request(:get, uri, params, headers, log, max_retries, max_retry_delay)
|
19
|
+
resp.body
|
20
|
+
end
|
21
|
+
|
22
|
+
# Performs a HEAD request and returns the response status as an integer.
|
23
|
+
# Note that unlike {Requester#get}, this does not raise an error in the
|
24
|
+
# event of an unsuccessful request.
|
25
|
+
#
|
26
|
+
# @param uri [URI, String] the URI to HEAD
|
27
|
+
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
28
|
+
# @param headers [Hash] the request headers.
|
29
|
+
# @param log [Boolean] whether to log each request URL and response code
|
30
|
+
# @return [Integer] the response code as an integer.
|
31
|
+
def head(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
|
32
|
+
head_response(uri, params: params, headers: headers, log: log, max_retries: max_retries, max_retry_delay: max_retry_delay).code
|
33
|
+
end
|
34
|
+
|
35
|
+
# Performs a GET request and returns the response, even in the event of
|
36
|
+
# a failed request.
|
37
|
+
#
|
38
|
+
# @param uri [URI, String] the URI to GET
|
39
|
+
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
40
|
+
# @param headers [Hash] the request headers.
|
41
|
+
# @param log [Boolean] whether to log each request URL and response code
|
42
|
+
# @return [RestClient::Response] the response
|
43
|
+
def get_response(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
|
44
|
+
make_request(:get, uri, params, headers, log, max_retries, max_retry_delay)
|
45
|
+
rescue RestClient::Exception => e
|
46
|
+
e.response
|
47
|
+
end
|
48
|
+
|
49
|
+
# Performs a HEAD request and returns the response, even in the event of
|
50
|
+
# a failed request.
|
51
|
+
#
|
52
|
+
# @param uri [URI, String] the URI to HEAD
|
53
|
+
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
54
|
+
# @param headers [Hash] the request headers.
|
55
|
+
# @param log [Boolean] whether to log each request URL and response code
|
56
|
+
# @return [RestClient::Response] the response
|
57
|
+
def head_response(uri, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
|
58
|
+
make_request(:head, uri, params, headers, log, max_retries, max_retry_delay)
|
59
|
+
rescue RestClient::Exception => e
|
60
|
+
e.response
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def make_request(method, url, params, headers, log, max_retries, max_retry_delay)
|
66
|
+
Requester.new(
|
67
|
+
method,
|
68
|
+
url,
|
69
|
+
params: params,
|
70
|
+
headers: headers,
|
71
|
+
log: log,
|
72
|
+
max_retries: max_retries,
|
73
|
+
max_retry_delay: max_retry_delay
|
74
|
+
).make_request
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
# rubocop:enable Metrics/ParameterLists
|
79
|
+
|
80
|
+
# ------------------------------------------------------------
|
81
|
+
# Class methods
|
82
|
+
|
83
|
+
class << self
|
84
|
+
include ClassMethods
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -1,108 +1,166 @@
|
|
1
|
+
require 'time'
|
1
2
|
require 'rest-client'
|
2
3
|
require 'berkeley_library/util/uris/appender'
|
4
|
+
require 'berkeley_library/util/uris/exceptions'
|
3
5
|
require 'berkeley_library/util/uris/validator'
|
6
|
+
require 'berkeley_library/util/uris/requester/class_methods'
|
4
7
|
require 'berkeley_library/logging'
|
5
8
|
|
6
9
|
module BerkeleyLibrary
|
7
10
|
module Util
|
8
11
|
module URIs
|
9
|
-
|
10
|
-
|
11
|
-
include BerkeleyLibrary::Logging
|
12
|
-
|
13
|
-
# Performs a GET request and returns the response body as a string.
|
14
|
-
#
|
15
|
-
# @param uri [URI, String] the URI to GET
|
16
|
-
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
17
|
-
# @param headers [Hash] the request headers.
|
18
|
-
# @return [String] the body as a string.
|
19
|
-
# @raise [RestClient::Exception] in the event of an unsuccessful request.
|
20
|
-
def get(uri, params: {}, headers: {})
|
21
|
-
resp = make_request(:get, uri, params, headers)
|
22
|
-
resp.body
|
23
|
-
end
|
12
|
+
class Requester
|
13
|
+
include BerkeleyLibrary::Logging
|
24
14
|
|
25
|
-
|
26
|
-
|
27
|
-
# event of an unsuccessful request.
|
28
|
-
#
|
29
|
-
# @param uri [URI, String] the URI to HEAD
|
30
|
-
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
31
|
-
# @param headers [Hash] the request headers.
|
32
|
-
# @return [Integer] the response code as an integer.
|
33
|
-
def head(uri, params: {}, headers: {})
|
34
|
-
head_response(uri, params: params, headers: headers).code
|
35
|
-
end
|
15
|
+
# ------------------------------------------------------------
|
16
|
+
# Constants
|
36
17
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
# @param headers [Hash] the request headers.
|
43
|
-
# @return [RestClient::Response] the body as a string.
|
44
|
-
def get_response(uri, params: {}, headers: {})
|
45
|
-
make_request(:get, uri, params, headers)
|
46
|
-
rescue RestClient::Exception => e
|
47
|
-
e.response
|
48
|
-
end
|
18
|
+
SUPPORTED_METHODS = %i[get head].freeze
|
19
|
+
RETRY_HEADER = :retry_after
|
20
|
+
RETRY_STATUSES = [429, 503].freeze
|
21
|
+
MAX_RETRY_DELAY_SECONDS = 10
|
22
|
+
MAX_RETRIES = 3
|
49
23
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
24
|
+
# ------------------------------------------------------------
|
25
|
+
# Attributes
|
26
|
+
|
27
|
+
attr_reader :method, :url_str, :headers, :log, :max_retries, :max_retry_delay
|
28
|
+
|
29
|
+
# ------------------------------------------------------------
|
30
|
+
# Initializer
|
31
|
+
|
32
|
+
# Initializes a new Requester.
|
33
|
+
#
|
34
|
+
# @param method [:get, :head] the HTTP method to use
|
35
|
+
# @param url [String, URI] the URL or URI to request
|
36
|
+
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
37
|
+
# @param headers [Hash] the request headers.
|
38
|
+
# @param log [Boolean] whether to log each request URL and response code
|
39
|
+
# @param max_retries [Integer] the maximum number of times to retry after a 429 or 503 with Retry-After
|
40
|
+
# @param max_retry_delay [Integer] the maximum retry delay (in seconds) to accept in a Retry-After header
|
41
|
+
# @raise URI::InvalidURIError if the specified URL is invalid
|
42
|
+
# rubocop:disable Metrics/ParameterLists
|
43
|
+
def initialize(method, url, params: {}, headers: {}, log: true, max_retries: MAX_RETRIES, max_retry_delay: MAX_RETRY_DELAY_SECONDS)
|
44
|
+
raise ArgumentError, "#{method} not supported" unless SUPPORTED_METHODS.include?(method)
|
45
|
+
raise ArgumentError, 'url cannot be nil' unless (uri = Validator.uri_or_nil(url))
|
46
|
+
|
47
|
+
@method = method
|
48
|
+
@url_str = url_str_with_params(uri, params)
|
49
|
+
@headers = headers
|
50
|
+
@log = log
|
51
|
+
@max_retries = max_retries
|
52
|
+
@max_retry_delay = max_retry_delay
|
53
|
+
end
|
54
|
+
|
55
|
+
# rubocop:enable Metrics/ParameterLists
|
62
56
|
|
63
|
-
|
57
|
+
# ------------------------------------------------------------
|
58
|
+
# Public instance methods
|
64
59
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
60
|
+
# @return [RestClient::Response]
|
61
|
+
def make_request
|
62
|
+
execute_request.tap do |resp|
|
63
|
+
log_response(resp)
|
69
64
|
end
|
65
|
+
rescue RestClient::Exception => e
|
66
|
+
log_response(e.response)
|
67
|
+
raise
|
68
|
+
end
|
69
|
+
|
70
|
+
# ------------------------------------------------------------
|
71
|
+
# Private methods
|
70
72
|
|
71
|
-
|
72
|
-
raise ArgumentError, 'url cannot be nil' unless (uri = Validator.uri_or_nil(url))
|
73
|
+
private
|
73
74
|
|
74
|
-
|
75
|
-
|
76
|
-
next if params.empty?
|
75
|
+
def log_response(response)
|
76
|
+
return unless log
|
77
77
|
|
78
|
-
|
79
|
-
|
80
|
-
end
|
78
|
+
logger.info("#{method.to_s.upcase} #{url_str} returned #{response.code}")
|
79
|
+
end
|
81
80
|
|
82
|
-
|
83
|
-
|
81
|
+
def url_str_with_params(uri, params)
|
82
|
+
elements = [uri]
|
83
|
+
if params.any?
|
84
|
+
elements << (uri.query ? '&' : '?')
|
85
|
+
elements << URI.encode_www_form(params)
|
84
86
|
end
|
85
87
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
88
|
+
Appender.new(*elements).to_url_str
|
89
|
+
end
|
90
|
+
|
91
|
+
def execute_request(retries_remaining = max_retries)
|
92
|
+
try_execute_request
|
93
|
+
rescue RestClient::Exception => e
|
94
|
+
response = e.response
|
95
|
+
raise unless (retry_delay = retry_delay_from(response))
|
96
|
+
|
97
|
+
wait_for_retry(response, retry_delay, retries_remaining)
|
98
|
+
execute_request(retries_remaining - 1)
|
99
|
+
end
|
100
|
+
|
101
|
+
def try_execute_request
|
102
|
+
RestClient::Request.execute(method: method, url: url_str, headers: headers).tap do |response|
|
103
|
+
# Not all failed RestClient requests throw exceptions
|
104
|
+
raise(exception_for(response)) unless response.code == 200
|
97
105
|
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def wait_for_retry(response, retry_delay, retries_remaining)
|
109
|
+
raise RetryLimitExceeded.new(response, max_retries: max_retries) unless retries_remaining > 0
|
110
|
+
raise RetryDelayTooLarge.new(response, delay: retry_delay, max_delay: max_retry_delay) if retry_delay > max_retry_delay
|
98
111
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
112
|
+
sleep(retry_delay)
|
113
|
+
end
|
114
|
+
|
115
|
+
def exception_for(resp)
|
116
|
+
status = resp.code
|
117
|
+
ex_class_for(status).new(resp, status).tap do |ex|
|
118
|
+
status_message = RestClient::STATUSES[status] || '(Unknown)'
|
119
|
+
ex.message = "#{status} #{status_message}"
|
104
120
|
end
|
105
121
|
end
|
122
|
+
|
123
|
+
def ex_class_for(status)
|
124
|
+
RestClient::Exceptions::EXCEPTIONS_MAP[status] || RestClient::RequestFailed
|
125
|
+
end
|
126
|
+
|
127
|
+
# Returns the retry interval for the specified exception, or `nil`
|
128
|
+
# if the response does not allow a retry.
|
129
|
+
#
|
130
|
+
# @param resp [RestClient::Response] the response
|
131
|
+
# @return [Integer, nil] the retry delay in seconds, or `nil` if the response
|
132
|
+
# does not allow a retry
|
133
|
+
def retry_delay_from(resp)
|
134
|
+
return unless RETRY_STATUSES.include?(resp.code)
|
135
|
+
return unless (retry_header_value = resp.headers[RETRY_HEADER])
|
136
|
+
return unless (retry_delay_seconds = parse_retry_header_value(retry_header_value))
|
137
|
+
|
138
|
+
[1, retry_delay_seconds.ceil].max
|
139
|
+
end
|
140
|
+
|
141
|
+
# @return [Float, nil] the retry delay in seconds, or `nil` if the delay cannot be parsed
|
142
|
+
def parse_retry_header_value(v)
|
143
|
+
# start by assuming it's a delay in seconds
|
144
|
+
Float(v) # should be an integer but let's not count on it
|
145
|
+
rescue ArgumentError
|
146
|
+
# assume it's an HTTP-date
|
147
|
+
parse_retry_after_date(v)
|
148
|
+
end
|
149
|
+
|
150
|
+
# Parses the specified RFC2822 datetime string and returns the interval between that
|
151
|
+
# datetime and the current time in seconds
|
152
|
+
#
|
153
|
+
# @param date_str [String] an RFC2822 datetime string
|
154
|
+
# @return [Float, nil] the interval between the current time and the specified datetime,
|
155
|
+
# or nil if `date_str` cannot be parsed
|
156
|
+
def parse_retry_after_date(date_str)
|
157
|
+
retry_after = DateTime.rfc2822(date_str).to_time
|
158
|
+
retry_after - Time.now
|
159
|
+
rescue ArgumentError
|
160
|
+
logger.warn("Can't parse #{RETRY_HEADER} value #{date_str}")
|
161
|
+
nil
|
162
|
+
end
|
163
|
+
|
106
164
|
end
|
107
165
|
end
|
108
166
|
end
|
@@ -31,10 +31,11 @@ module BerkeleyLibrary
|
|
31
31
|
# @param uri [URI, String] the URI to GET
|
32
32
|
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
33
33
|
# @param headers [Hash] the request headers.
|
34
|
+
# @param log [Boolean] whether to log each request URL and response code
|
34
35
|
# @return [String] the body as a string.
|
35
36
|
# @raise [RestClient::Exception] in the event of an unsuccessful request.
|
36
|
-
def get(uri, params: {}, headers: {})
|
37
|
-
Requester.get(uri, params: params, headers: headers)
|
37
|
+
def get(uri, params: {}, headers: {}, log: true)
|
38
|
+
Requester.get(uri, params: params, headers: headers, log: log)
|
38
39
|
end
|
39
40
|
|
40
41
|
# Performs a HEAD request and returns the response status as an integer.
|
@@ -44,9 +45,10 @@ module BerkeleyLibrary
|
|
44
45
|
# @param uri [URI, String] the URI to HEAD
|
45
46
|
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
46
47
|
# @param headers [Hash] the request headers.
|
48
|
+
# @param log [Boolean] whether to log each request URL and response code
|
47
49
|
# @return [Integer] the response code as an integer.
|
48
|
-
def head(uri, params: {}, headers: {})
|
49
|
-
Requester.head(uri, params: params, headers: headers)
|
50
|
+
def head(uri, params: {}, headers: {}, log: true)
|
51
|
+
Requester.head(uri, params: params, headers: headers, log: log)
|
50
52
|
end
|
51
53
|
|
52
54
|
# Performs a GET request and returns the response, even in the event of
|
@@ -55,9 +57,10 @@ module BerkeleyLibrary
|
|
55
57
|
# @param uri [URI, String] the URI to GET
|
56
58
|
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
57
59
|
# @param headers [Hash] the request headers.
|
58
|
-
# @
|
59
|
-
|
60
|
-
|
60
|
+
# @param log [Boolean] whether to log each request URL and response code
|
61
|
+
# @return [RestClient::Response] the response
|
62
|
+
def get_response(uri, params: {}, headers: {}, log: true)
|
63
|
+
Requester.get_response(uri, params: params, headers: headers, log: log)
|
61
64
|
end
|
62
65
|
|
63
66
|
# Performs a HEAD request and returns the response, even in the event of
|
@@ -66,9 +69,10 @@ module BerkeleyLibrary
|
|
66
69
|
# @param uri [URI, String] the URI to HEAD
|
67
70
|
# @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
|
68
71
|
# @param headers [Hash] the request headers.
|
69
|
-
# @
|
70
|
-
|
71
|
-
|
72
|
+
# @param log [Boolean] whether to log each request URL and response code
|
73
|
+
# @return [RestClient::Response] the response
|
74
|
+
def head_response(uri, params: {}, headers: {}, log: true)
|
75
|
+
Requester.head_response(uri, params: params, headers: headers, log: log)
|
72
76
|
end
|
73
77
|
|
74
78
|
# Returns the specified URL as a URI, or `nil` if the URL is `nil`.
|
@@ -65,6 +65,250 @@ module BerkeleyLibrary
|
|
65
65
|
result = Requester.get(url1)
|
66
66
|
expect(result).to eq(expected_body)
|
67
67
|
end
|
68
|
+
|
69
|
+
describe 'retries' do
|
70
|
+
let(:url) { 'https://example.org/' }
|
71
|
+
let(:expected_body) { 'Help! I am trapped in a unit test' }
|
72
|
+
|
73
|
+
context 'handling 429 Too Many Requests' do
|
74
|
+
context 'with Retry-After' do
|
75
|
+
context 'in seconds' do
|
76
|
+
it 'retries after the specified delay' do
|
77
|
+
retry_after_seconds = 1
|
78
|
+
|
79
|
+
stub_request(:get, url)
|
80
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
81
|
+
.to_return(status: 200, body: expected_body)
|
82
|
+
|
83
|
+
requester = Requester.new(:get, url)
|
84
|
+
expect(requester).to receive(:sleep).with(1).once
|
85
|
+
|
86
|
+
result = requester.make_request
|
87
|
+
expect(result).to eq(expected_body)
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'handles a non-integer retry delay' do
|
91
|
+
retry_after_seconds = 1.5
|
92
|
+
stub_request(:get, url)
|
93
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
94
|
+
.to_return(status: 200, body: expected_body)
|
95
|
+
|
96
|
+
requester = Requester.new(:get, url)
|
97
|
+
expect(requester).to receive(:sleep).with(2).once
|
98
|
+
|
99
|
+
result = requester.make_request
|
100
|
+
expect(result).to eq(expected_body)
|
101
|
+
end
|
102
|
+
|
103
|
+
it 'raises RetryDelayTooLarge if the delay is too large' do
|
104
|
+
retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
|
105
|
+
|
106
|
+
stub_request(:get, url)
|
107
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
108
|
+
|
109
|
+
requester = Requester.new(:get, url)
|
110
|
+
expect(requester).not_to receive(:sleep)
|
111
|
+
|
112
|
+
expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
|
113
|
+
expect(ex.cause).to be_a(RestClient::TooManyRequests)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'raises RetryLimitExceeded if there are too many retries' do
|
118
|
+
retry_after_seconds = 1
|
119
|
+
|
120
|
+
stub_request(:get, url)
|
121
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
122
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
123
|
+
|
124
|
+
requester = Requester.new(:get, url, max_retries: 1)
|
125
|
+
expect(requester).to receive(:sleep).with(1).once
|
126
|
+
|
127
|
+
expect { requester.make_request }.to raise_error(RetryLimitExceeded) do |ex|
|
128
|
+
expect(ex.cause).to be_a(RestClient::TooManyRequests)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
context 'as RFC2822 datetime' do
|
134
|
+
it 'retries after the specified delay' do
|
135
|
+
retry_after_seconds = 1
|
136
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
137
|
+
|
138
|
+
stub_request(:get, url)
|
139
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
140
|
+
.to_return(status: 200, body: expected_body)
|
141
|
+
|
142
|
+
requester = Requester.new(:get, url)
|
143
|
+
expect(requester).to receive(:sleep).with(1).once
|
144
|
+
|
145
|
+
result = requester.make_request
|
146
|
+
expect(result).to eq(expected_body)
|
147
|
+
end
|
148
|
+
|
149
|
+
it 'handles a non-integer retry delay' do
|
150
|
+
retry_after_seconds = 2.75
|
151
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
152
|
+
|
153
|
+
stub_request(:get, url)
|
154
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
155
|
+
.to_return(status: 200, body: expected_body)
|
156
|
+
|
157
|
+
requester = Requester.new(:get, url)
|
158
|
+
expected_value = a_value_within(1).of(retry_after_seconds)
|
159
|
+
expect(requester).to receive(:sleep).with(expected_value).once
|
160
|
+
|
161
|
+
result = requester.make_request
|
162
|
+
expect(result).to eq(expected_body)
|
163
|
+
end
|
164
|
+
|
165
|
+
it 'raises RetryDelayTooLarge if the delay is too large' do
|
166
|
+
retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
|
167
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
168
|
+
|
169
|
+
stub_request(:get, url)
|
170
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
171
|
+
|
172
|
+
requester = Requester.new(:get, url)
|
173
|
+
expect(requester).not_to receive(:sleep)
|
174
|
+
|
175
|
+
expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
|
176
|
+
expect(ex.cause).to be_a(RestClient::TooManyRequests)
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
it 'raises RetryLimitExceeded if there are too many retries' do
|
181
|
+
retry_after_seconds = 1
|
182
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
183
|
+
|
184
|
+
stub_request(:get, url)
|
185
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
186
|
+
.to_return(status: 429, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
187
|
+
|
188
|
+
requester = Requester.new(:get, url, max_retries: 1)
|
189
|
+
expect(requester).to receive(:sleep).with(1).once
|
190
|
+
|
191
|
+
expect { requester.make_request }.to raise_error(RetryLimitExceeded) do |ex|
|
192
|
+
expect(ex.cause).to be_a(RestClient::TooManyRequests)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
end
|
197
|
+
|
198
|
+
it 'ignores an invalid Retry-After' do
|
199
|
+
stub_request(:get, url)
|
200
|
+
.to_return(status: 429, headers: { 'Retry-After' => 'the end of the world' })
|
201
|
+
|
202
|
+
requester = Requester.new(:get, url)
|
203
|
+
expect { requester.make_request }.to raise_error(RestClient::TooManyRequests)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
context 'handling 503 Service Unavailable' do
|
209
|
+
context 'with Retry-After' do
|
210
|
+
context 'in seconds' do
|
211
|
+
it 'retries after the specified delay' do
|
212
|
+
retry_after_seconds = 1
|
213
|
+
|
214
|
+
stub_request(:get, url)
|
215
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
216
|
+
.to_return(status: 200, body: expected_body)
|
217
|
+
|
218
|
+
requester = Requester.new(:get, url)
|
219
|
+
expect(requester).to receive(:sleep).with(1).once
|
220
|
+
|
221
|
+
result = requester.make_request
|
222
|
+
expect(result).to eq(expected_body)
|
223
|
+
end
|
224
|
+
|
225
|
+
it 'handles a non-integer retry delay' do
|
226
|
+
retry_after_seconds = 0.75
|
227
|
+
stub_request(:get, url)
|
228
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
229
|
+
.to_return(status: 200, body: expected_body)
|
230
|
+
|
231
|
+
requester = Requester.new(:get, url)
|
232
|
+
expect(requester).to receive(:sleep).with(1).once
|
233
|
+
|
234
|
+
result = requester.make_request
|
235
|
+
expect(result).to eq(expected_body)
|
236
|
+
end
|
237
|
+
|
238
|
+
it 'raises RetryDelayTooLarge if the delay is too large' do
|
239
|
+
retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
|
240
|
+
|
241
|
+
stub_request(:get, url)
|
242
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_seconds.to_s })
|
243
|
+
|
244
|
+
requester = Requester.new(:get, url)
|
245
|
+
expect(requester).not_to receive(:sleep)
|
246
|
+
|
247
|
+
expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
|
248
|
+
expect(ex.cause).to be_a(RestClient::ServiceUnavailable)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
context 'as RFC2822 datetime' do
|
254
|
+
it 'retries after the specified delay' do
|
255
|
+
retry_after_seconds = 1
|
256
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
257
|
+
|
258
|
+
stub_request(:get, url)
|
259
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
260
|
+
.to_return(status: 200, body: expected_body)
|
261
|
+
|
262
|
+
requester = Requester.new(:get, url)
|
263
|
+
expect(requester).to receive(:sleep).with(1).once
|
264
|
+
|
265
|
+
result = requester.make_request
|
266
|
+
expect(result).to eq(expected_body)
|
267
|
+
end
|
268
|
+
|
269
|
+
it 'handles a non-integer retry delay' do
|
270
|
+
retry_after_seconds = 1.75
|
271
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
272
|
+
|
273
|
+
stub_request(:get, url)
|
274
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
275
|
+
.to_return(status: 200, body: expected_body)
|
276
|
+
|
277
|
+
requester = Requester.new(:get, url)
|
278
|
+
expected_value = a_value_within(1).of(retry_after_seconds)
|
279
|
+
expect(requester).to receive(:sleep).with(expected_value).once
|
280
|
+
|
281
|
+
result = requester.make_request
|
282
|
+
expect(result).to eq(expected_body)
|
283
|
+
end
|
284
|
+
|
285
|
+
it 'raises RetryDelayTooLarge if the delay is too large' do
|
286
|
+
retry_after_seconds = 1 + BerkeleyLibrary::Util::URIs::Requester::MAX_RETRY_DELAY_SECONDS
|
287
|
+
retry_after_datetime = (Time.now + retry_after_seconds)
|
288
|
+
|
289
|
+
stub_request(:get, url)
|
290
|
+
.to_return(status: 503, headers: { 'Retry-After' => retry_after_datetime.rfc2822 })
|
291
|
+
|
292
|
+
requester = Requester.new(:get, url)
|
293
|
+
expect(requester).not_to receive(:sleep)
|
294
|
+
|
295
|
+
expect { requester.make_request }.to raise_error(RetryDelayTooLarge) do |ex|
|
296
|
+
expect(ex.cause).to be_a(RestClient::ServiceUnavailable)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
it 'ignores an invalid Retry-After' do
|
302
|
+
stub_request(:get, url)
|
303
|
+
.to_return(status: 503, headers: { 'Retry-After' => 'the end of the world' })
|
304
|
+
|
305
|
+
requester = Requester.new(:get, url)
|
306
|
+
expect { requester.make_request }.to raise_error(RestClient::ServiceUnavailable)
|
307
|
+
end
|
308
|
+
end
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
68
312
|
end
|
69
313
|
|
70
314
|
describe :head do
|
@@ -145,6 +389,112 @@ module BerkeleyLibrary
|
|
145
389
|
expect { Requester.head(nil) }.to raise_error(ArgumentError)
|
146
390
|
end
|
147
391
|
end
|
392
|
+
|
393
|
+
describe 'logging' do
|
394
|
+
attr_reader :logger
|
395
|
+
|
396
|
+
before do
|
397
|
+
@logger = instance_double(BerkeleyLibrary::Logging::Logger)
|
398
|
+
allow(BerkeleyLibrary::Logging).to receive(:logger).and_return(logger)
|
399
|
+
end
|
400
|
+
|
401
|
+
context 'GET' do
|
402
|
+
it 'logs request URLs and response codes for successful GET requests' do
|
403
|
+
url = 'https://example.org/'
|
404
|
+
expected_body = 'Help! I am trapped in a unit test'
|
405
|
+
stub_request(:get, url).to_return(body: expected_body)
|
406
|
+
|
407
|
+
expect(logger).to receive(:info).with(/#{url}.*200/)
|
408
|
+
Requester.send(:get, url)
|
409
|
+
end
|
410
|
+
|
411
|
+
it 'can suppress logging for successful GET requests' do
|
412
|
+
url = 'https://example.org/'
|
413
|
+
expected_body = 'Help! I am trapped in a unit test'
|
414
|
+
stub_request(:get, url).to_return(body: expected_body)
|
415
|
+
|
416
|
+
expect(logger).not_to receive(:info)
|
417
|
+
Requester.send(:get, url, log: false)
|
418
|
+
end
|
419
|
+
|
420
|
+
it 'logs request URLs and response codes for failed GET requests' do
|
421
|
+
url = 'https://example.org/'
|
422
|
+
status = 500
|
423
|
+
stub_request(:get, url).to_return(status: status)
|
424
|
+
|
425
|
+
expect(logger).to receive(:info).with(/#{url}.*#{status}/)
|
426
|
+
expect { Requester.send(:get, url) }.to raise_error(RestClient::InternalServerError)
|
427
|
+
end
|
428
|
+
|
429
|
+
it 'can suppress logging for failed GET requests' do
|
430
|
+
url = 'https://example.org/'
|
431
|
+
stub_request(:get, url).to_return(status: 500)
|
432
|
+
|
433
|
+
expect(logger).not_to receive(:info)
|
434
|
+
expect { Requester.send(:get, url, log: false) }.to raise_error(RestClient::InternalServerError)
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
context 'HEAD' do
|
439
|
+
it 'logs request URLs and response codes for successful HEAD requests' do
|
440
|
+
url = 'https://example.org/'
|
441
|
+
expected_body = 'Help! I am trapped in a unit test'
|
442
|
+
stub_request(:head, url).to_return(body: expected_body)
|
443
|
+
|
444
|
+
expect(logger).to receive(:info).with(/#{url}.*200/)
|
445
|
+
Requester.send(:head, url)
|
446
|
+
end
|
447
|
+
|
448
|
+
it 'can suppress logging for successful HEAD requests' do
|
449
|
+
url = 'https://example.org/'
|
450
|
+
expected_body = 'Help! I am trapped in a unit test'
|
451
|
+
stub_request(:head, url).to_return(body: expected_body)
|
452
|
+
|
453
|
+
expect(logger).not_to receive(:info)
|
454
|
+
Requester.send(:head, url, log: false)
|
455
|
+
end
|
456
|
+
|
457
|
+
it 'logs request URLs and response codes for failed HEAD requests' do
|
458
|
+
url = 'https://example.org/'
|
459
|
+
status = 500
|
460
|
+
stub_request(:head, url).to_return(status: status)
|
461
|
+
|
462
|
+
expect(logger).to receive(:info).with(/#{url}.*#{status}/)
|
463
|
+
expect(Requester.send(:head, url)).to eq(status)
|
464
|
+
end
|
465
|
+
|
466
|
+
it 'can suppress logging for failed HEAD requests' do
|
467
|
+
url = 'https://example.org/'
|
468
|
+
status = 500
|
469
|
+
stub_request(:head, url).to_return(status: status)
|
470
|
+
|
471
|
+
expect(logger).not_to receive(:info)
|
472
|
+
expect(Requester.send(:head, url, log: false)).to eq(status)
|
473
|
+
end
|
474
|
+
end
|
475
|
+
end
|
476
|
+
|
477
|
+
describe :new do
|
478
|
+
it 'rejects invalid URIs' do
|
479
|
+
url = 'not a uri'
|
480
|
+
Requester::SUPPORTED_METHODS.each do |method|
|
481
|
+
expect { Requester.new(method, url) }.to raise_error(URI::InvalidURIError)
|
482
|
+
end
|
483
|
+
end
|
484
|
+
|
485
|
+
it 'rejects nil URIs' do
|
486
|
+
Requester::SUPPORTED_METHODS.each do |method|
|
487
|
+
expect { Requester.new(method, nil) }.to raise_error(ArgumentError)
|
488
|
+
end
|
489
|
+
end
|
490
|
+
|
491
|
+
it 'rejects unsupported methods' do
|
492
|
+
url = 'https://example.org/'
|
493
|
+
%i[put patch post].each do |method|
|
494
|
+
expect { Requester.new(method, url) }.to raise_error(ArgumentError)
|
495
|
+
end
|
496
|
+
end
|
497
|
+
end
|
148
498
|
end
|
149
499
|
end
|
150
500
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: berkeley_library-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Moles
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-03-
|
11
|
+
date: 2023-03-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: berkeley_library-logging
|
@@ -241,7 +241,9 @@ files:
|
|
241
241
|
- lib/berkeley_library/util/times.rb
|
242
242
|
- lib/berkeley_library/util/uris.rb
|
243
243
|
- lib/berkeley_library/util/uris/appender.rb
|
244
|
+
- lib/berkeley_library/util/uris/exceptions.rb
|
244
245
|
- lib/berkeley_library/util/uris/requester.rb
|
246
|
+
- lib/berkeley_library/util/uris/requester/class_methods.rb
|
245
247
|
- lib/berkeley_library/util/uris/validator.rb
|
246
248
|
- rakelib/.rubocop.yml
|
247
249
|
- rakelib/coverage.rake
|