sqewer 7.0.0 → 8.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGELOG.md +13 -0
- data/lib/sqewer.rb +2 -16
- data/lib/sqewer/connection.rb +44 -14
- data/lib/sqewer/version.rb +1 -1
- data/sqewer.gemspec +1 -0
- metadata +20 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04eca66282554d24d485eb24c83cfbbe6c920eeafe2cc87f2f4638fb1c35ef82
|
4
|
+
data.tar.gz: 8eadd3fdbc63b6f0f26a65d5338a587fd6819889d94a83020914f4af65e527ff
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27504ff6d9db93063d766ced7efc1b76de5f4e7977490a2046316438eade94389705a75470e49d9d8e8a5a05862b2551df7d6d673398938b12c166842e6380bf
|
7
|
+
data.tar.gz: 1b1a9ef385a4ed693cc6d0999eb8d526b36fda101cfc6ab19e11d856ddf0327a4237affe854e7e03891c16cb9f8e7f4fe4c7140b26218eef64bd23909d4817c2
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
### 8.0.3
|
2
|
+
- Fix `Sqewer::Connection` to properly get the list of AWS errors
|
3
|
+
|
4
|
+
### 8.0.2
|
5
|
+
- Add `Aws::SQS::Errors::AccessDenied` to the list of errors that Sqewer must release the singleton SQS client
|
6
|
+
|
7
|
+
### 8.0.1
|
8
|
+
- Release the singleton SQS client when AWS raises credentials error to be able to use a new credential next time
|
9
|
+
|
10
|
+
### 8.0.0
|
11
|
+
- Remove method `Sqewer.client=`
|
12
|
+
- Change `Sqewer::Connection` to use a singleton SQS client by default
|
13
|
+
|
1
14
|
### 7.0.0
|
2
15
|
- Remove support of Ruby 2.3, 2.4 and 2.5
|
3
16
|
- Remove support of Rails 4
|
data/lib/sqewer.rb
CHANGED
@@ -11,23 +11,9 @@ module Sqewer
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
#
|
15
|
-
# We recommend setting the options instance_profile_credentials_timeout and
|
16
|
-
# instance_profile_credentials_retries, for example:
|
17
|
-
#
|
18
|
-
# sqs_client = Aws::SQS::Client.new(
|
19
|
-
# instance_profile_credentials_timeout: 1,
|
20
|
-
# instance_profile_credentials_retries: 5,
|
21
|
-
# )
|
22
|
-
# Storm.client = sqs_client
|
23
|
-
#
|
24
|
-
# @param client[Aws::SQS::Client] an instance of Aws::SQS::Client
|
25
|
-
def self.client=(client)
|
26
|
-
@client = client
|
27
|
-
end
|
28
|
-
|
14
|
+
# Returns a singleton of Aws::SQS::Client
|
29
15
|
def self.client
|
30
|
-
|
16
|
+
Sqewer::Connection.client
|
31
17
|
end
|
32
18
|
|
33
19
|
# Loads a particular Sqewer extension that is not loaded
|
data/lib/sqewer/connection.rb
CHANGED
@@ -39,12 +39,25 @@ class Sqewer::Connection
|
|
39
39
|
raise "SQS_QUEUE_URL not set in the environment. This is the queue URL Sqewer uses by default."
|
40
40
|
end
|
41
41
|
|
42
|
+
# Returns a singleton of Aws::SQS::Client
|
43
|
+
def self.client
|
44
|
+
# It's better using a singleton client to prevent making a lot of HTTP
|
45
|
+
# requests to the AWS metadata endpoint when getting credentials.
|
46
|
+
@client ||= begin
|
47
|
+
require 'aws-sdk-sqs'
|
48
|
+
::Aws::SQS::Client.new(
|
49
|
+
instance_profile_credentials_timeout: 1,
|
50
|
+
instance_profile_credentials_retries: 5,
|
51
|
+
)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
42
55
|
# Initializes a new adapter, with access to the SQS queue at the given URL.
|
43
56
|
#
|
44
57
|
# @param queue_url[String] the SQS queue URL (the URL can be copied from your AWS console)
|
45
|
-
def initialize(queue_url)
|
46
|
-
require 'aws-sdk-sqs'
|
58
|
+
def initialize(queue_url, client: self.class.client)
|
47
59
|
@queue_url = queue_url
|
60
|
+
@client = client
|
48
61
|
end
|
49
62
|
|
50
63
|
# Receive at most 10 messages from the queue, and return the array of Message objects. Retries for at
|
@@ -55,7 +68,7 @@ class Sqewer::Connection
|
|
55
68
|
# @return [Array<Message>] an array of Message objects
|
56
69
|
def receive_messages
|
57
70
|
Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_RECEIVE_FAILURES do
|
58
|
-
response = client.receive_message(
|
71
|
+
response = @client.receive_message(
|
59
72
|
queue_url: @queue_url,
|
60
73
|
attribute_names: ['All'],
|
61
74
|
wait_time_seconds: DEFAULT_TIMEOUT_SECONDS,
|
@@ -63,6 +76,14 @@ class Sqewer::Connection
|
|
63
76
|
)
|
64
77
|
response.messages.map {|message| Message.new(message.receipt_handle, message.body, message.attributes) }
|
65
78
|
end
|
79
|
+
rescue *sqs_errors_to_release_client
|
80
|
+
# We noticed cases where errors related to AWS credentials started to happen suddenly.
|
81
|
+
# We don't know the root cause yet, but what we can do is release the
|
82
|
+
# singleton @client instance because it contains a cache of credentials that in most
|
83
|
+
# cases is no longer valid.
|
84
|
+
self.class.release_client
|
85
|
+
|
86
|
+
raise
|
66
87
|
end
|
67
88
|
|
68
89
|
# Send a message to the backing queue
|
@@ -189,6 +210,12 @@ class Sqewer::Connection
|
|
189
210
|
buffer.each_batch {|batch| handle_batch_with_retries(:delete_message_batch, batch) }
|
190
211
|
end
|
191
212
|
|
213
|
+
protected
|
214
|
+
|
215
|
+
def self.release_client
|
216
|
+
@client = nil
|
217
|
+
end
|
218
|
+
|
192
219
|
private
|
193
220
|
|
194
221
|
def network_and_aws_sdk_errors
|
@@ -197,7 +224,7 @@ class Sqewer::Connection
|
|
197
224
|
|
198
225
|
def handle_batch_with_retries(method, batch)
|
199
226
|
Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_FAILURES_PER_CALL do
|
200
|
-
resp = client.send(method, queue_url: @queue_url, entries: batch)
|
227
|
+
resp = @client.send(method, queue_url: @queue_url, entries: batch)
|
201
228
|
wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
|
202
229
|
if wrong_messages.any?
|
203
230
|
err = wrong_messages.inspect + ', ' + resp.inspect
|
@@ -208,17 +235,20 @@ class Sqewer::Connection
|
|
208
235
|
raise NotOurFaultAwsError
|
209
236
|
end
|
210
237
|
end
|
238
|
+
rescue *sqs_errors_to_release_client
|
239
|
+
# We noticed cases where errors related to AWS credentials started to happen suddenly.
|
240
|
+
# We don't know the root cause yet, but what we can do is release the
|
241
|
+
# singleton @client instance because it contains a cache of credentials that in most
|
242
|
+
# cases is no longer valid.
|
243
|
+
self.class.release_client
|
244
|
+
|
245
|
+
raise
|
211
246
|
end
|
212
247
|
|
213
|
-
def
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
@client ||= Aws::SQS::Client.new(
|
220
|
-
instance_profile_credentials_timeout: 1, # defaults to 1 second
|
221
|
-
instance_profile_credentials_retries: 5, # defaults to 0 retries
|
222
|
-
)
|
248
|
+
def sqs_errors_to_release_client
|
249
|
+
[
|
250
|
+
Aws::Errors::MissingCredentialsError,
|
251
|
+
Aws::SQS::Errors::AccessDenied,
|
252
|
+
]
|
223
253
|
end
|
224
254
|
end
|
data/lib/sqewer/version.rb
CHANGED
data/sqewer.gemspec
CHANGED
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqewer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 8.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Julik Tarkhanov
|
8
8
|
- Andrei Horak
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-07-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk-sqs
|
@@ -221,6 +221,20 @@ dependencies:
|
|
221
221
|
- - "~>"
|
222
222
|
- !ruby/object:Gem::Version
|
223
223
|
version: '2'
|
224
|
+
- !ruby/object:Gem::Dependency
|
225
|
+
name: pry-byebug
|
226
|
+
requirement: !ruby/object:Gem::Requirement
|
227
|
+
requirements:
|
228
|
+
- - ">="
|
229
|
+
- !ruby/object:Gem::Version
|
230
|
+
version: '0'
|
231
|
+
type: :development
|
232
|
+
prerelease: false
|
233
|
+
version_requirements: !ruby/object:Gem::Requirement
|
234
|
+
requirements:
|
235
|
+
- - ">="
|
236
|
+
- !ruby/object:Gem::Version
|
237
|
+
version: '0'
|
224
238
|
description: A full-featured library for all them SQS worker needs
|
225
239
|
email:
|
226
240
|
- me@julik.nl
|
@@ -271,7 +285,7 @@ homepage: https://github.com/WeTransfer/sqewer
|
|
271
285
|
licenses: []
|
272
286
|
metadata:
|
273
287
|
allowed_push_host: https://rubygems.org
|
274
|
-
post_install_message:
|
288
|
+
post_install_message:
|
275
289
|
rdoc_options: []
|
276
290
|
require_paths:
|
277
291
|
- lib
|
@@ -286,8 +300,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
286
300
|
- !ruby/object:Gem::Version
|
287
301
|
version: '0'
|
288
302
|
requirements: []
|
289
|
-
rubygems_version: 3.
|
290
|
-
signing_key:
|
303
|
+
rubygems_version: 3.1.6
|
304
|
+
signing_key:
|
291
305
|
specification_version: 4
|
292
306
|
summary: Process jobs from SQS
|
293
307
|
test_files: []
|