aws-sdk-rails 3.9.1 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb +78 -0
- data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +28 -39
- data/lib/aws/rails/sqs_active_job/configuration.rb +12 -1
- data/lib/aws/rails/sqs_active_job/executor.rb +11 -2
- data/lib/aws/rails/sqs_active_job/job_runner.rb +5 -0
- data/lib/aws/rails/sqs_active_job/poller.rb +14 -2
- data/lib/aws-sdk-rails.rb +1 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b6ff9a7f26db95b0ae83526c594bb985b991da455979daf7d2a6ea92be0a5187
|
4
|
+
data.tar.gz: bbb91361034021720e0f43058777b530212d6c43d489f23999b636dd7e187edb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a28a3bf351332f237ef4ce23bbcdb6115ac41468d14dd3d2e1eb76c7a395c33697c9f8c55412fdad3a2b4fc63cbf7dca350243bdf5651743a7ccadfb9adebb0a
|
7
|
+
data.tar.gz: 80076e9047b439cdbd4fe8fe2da651b3cc9d23ed3346fb22c32724b9b3e00463dd77a52f99934bfe26c79ce6d9cc9a17d74fdf6683c9dc5b85e52c40a80f0d9d
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.11.0
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveJob
|
4
|
+
module QueueAdapters
|
5
|
+
class AmazonSqsAdapter
|
6
|
+
# == build request parameter of Aws::SQS::Client
|
7
|
+
class Params
|
8
|
+
class << self
|
9
|
+
def assured_delay_seconds(timestamp)
|
10
|
+
delay = (timestamp - Time.now.to_f).floor
|
11
|
+
delay = 0 if delay.negative?
|
12
|
+
raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
|
13
|
+
|
14
|
+
delay
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(job, body)
|
19
|
+
@job = job
|
20
|
+
@body = body || job.serialize
|
21
|
+
end
|
22
|
+
|
23
|
+
def queue_url
|
24
|
+
@queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def entry
|
28
|
+
if Aws::Rails::SqsActiveJob.fifo?(queue_url)
|
29
|
+
default_entry.merge(options_for_fifo)
|
30
|
+
else
|
31
|
+
default_entry
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def default_entry
|
38
|
+
{
|
39
|
+
message_body: Aws::Json.dump(@body),
|
40
|
+
message_attributes: message_attributes
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def message_attributes
|
45
|
+
{
|
46
|
+
'aws_sqs_active_job_class' => {
|
47
|
+
string_value: @job.class.to_s,
|
48
|
+
data_type: 'String'
|
49
|
+
},
|
50
|
+
'aws_sqs_active_job_version' => {
|
51
|
+
string_value: Aws::Rails::VERSION,
|
52
|
+
data_type: 'String'
|
53
|
+
}
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def options_for_fifo
|
58
|
+
options = {}
|
59
|
+
options[:message_deduplication_id] =
|
60
|
+
Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
|
61
|
+
|
62
|
+
message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
|
63
|
+
message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
|
64
|
+
|
65
|
+
options[:message_group_id] = message_group_id
|
66
|
+
options
|
67
|
+
end
|
68
|
+
|
69
|
+
def deduplication_body
|
70
|
+
ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
|
71
|
+
ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
|
72
|
+
|
73
|
+
@body.except(*ex_dedup_keys)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -10,55 +10,44 @@ module ActiveJob
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def enqueue_at(job, timestamp)
|
13
|
-
delay = (timestamp
|
14
|
-
|
15
|
-
delay = 0 if delay.negative?
|
16
|
-
raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
|
17
|
-
|
13
|
+
delay = Params.assured_delay_seconds(timestamp)
|
18
14
|
_enqueue(job, nil, delay_seconds: delay)
|
19
15
|
end
|
20
16
|
|
17
|
+
def enqueue_all(jobs)
|
18
|
+
enqueued_count = 0
|
19
|
+
jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
|
20
|
+
queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
|
21
|
+
base_send_message_opts = { queue_url: queue_url }
|
22
|
+
|
23
|
+
same_queue_jobs.each_slice(10) do |chunk|
|
24
|
+
entries = chunk.map do |job|
|
25
|
+
entry = Params.new(job, nil).entry
|
26
|
+
entry[:id] = job.job_id
|
27
|
+
entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
|
28
|
+
entry
|
29
|
+
end
|
30
|
+
|
31
|
+
send_message_opts = base_send_message_opts.deep_dup
|
32
|
+
send_message_opts[:entries] = entries
|
33
|
+
|
34
|
+
send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
|
35
|
+
enqueued_count += send_message_batch_result.successful.count
|
36
|
+
end
|
37
|
+
end
|
38
|
+
enqueued_count
|
39
|
+
end
|
40
|
+
|
21
41
|
private
|
22
42
|
|
23
43
|
def _enqueue(job, body = nil, send_message_opts = {})
|
24
44
|
body ||= job.serialize
|
25
|
-
|
26
|
-
send_message_opts
|
27
|
-
send_message_opts[:
|
28
|
-
send_message_opts[:message_attributes] = message_attributes(job)
|
29
|
-
|
30
|
-
if Aws::Rails::SqsActiveJob.fifo?(queue_url)
|
31
|
-
send_message_opts[:message_deduplication_id] =
|
32
|
-
Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body(job, body)))
|
33
|
-
|
34
|
-
message_group_id = job.message_group_id if job.respond_to?(:message_group_id)
|
35
|
-
message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
|
36
|
-
|
37
|
-
send_message_opts[:message_group_id] = message_group_id
|
38
|
-
end
|
45
|
+
params = Params.new(job, body)
|
46
|
+
send_message_opts = send_message_opts.merge(params.entry)
|
47
|
+
send_message_opts[:queue_url] = params.queue_url
|
39
48
|
|
40
49
|
Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
|
41
50
|
end
|
42
|
-
|
43
|
-
def message_attributes(job)
|
44
|
-
{
|
45
|
-
'aws_sqs_active_job_class' => {
|
46
|
-
string_value: job.class.to_s,
|
47
|
-
data_type: 'String'
|
48
|
-
},
|
49
|
-
'aws_sqs_active_job_version' => {
|
50
|
-
string_value: Aws::Rails::VERSION,
|
51
|
-
data_type: 'String'
|
52
|
-
}
|
53
|
-
}
|
54
|
-
end
|
55
|
-
|
56
|
-
def deduplication_body(job, body)
|
57
|
-
ex_dedup_keys = job.excluded_deduplication_keys if job.respond_to?(:excluded_deduplication_keys)
|
58
|
-
ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
|
59
|
-
|
60
|
-
body.except(*ex_dedup_keys)
|
61
|
-
end
|
62
51
|
end
|
63
52
|
|
64
53
|
# create an alias to allow `:amazon` to be used as the adapter name
|
@@ -25,6 +25,7 @@ module Aws
|
|
25
25
|
DEFAULTS = {
|
26
26
|
max_messages: 10,
|
27
27
|
shutdown_timeout: 15,
|
28
|
+
retry_standard_errors: true, # TODO: Remove in next MV
|
28
29
|
queues: {},
|
29
30
|
logger: ::Rails.logger,
|
30
31
|
message_group_id: 'SqsActiveJobGroup',
|
@@ -64,6 +65,16 @@ module Aws
|
|
64
65
|
# will not be deleted from the SQS queue and will be retryable after
|
65
66
|
# the visibility timeout.
|
66
67
|
#
|
68
|
+
# @ option options [Boolean] :retry_standard_errors
|
69
|
+
# If `true`, StandardErrors raised by ActiveJobs are left on the queue
|
70
|
+
# and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
|
71
|
+
# This behavior overrides the standard Rails ActiveJob
|
72
|
+
# [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
|
73
|
+
# behavior. When set to `true` the retries provided by this will be
|
74
|
+
# on top of any retries configured on the job with `retry_on`.
|
75
|
+
# When `false`, retry behavior is fully configured
|
76
|
+
# through `retry_on`/`discard_on` on the ActiveJobs.
|
77
|
+
#
|
67
78
|
# @option options [ActiveSupport::Logger] :logger Logger to use
|
68
79
|
# for the poller.
|
69
80
|
#
|
@@ -126,7 +137,7 @@ module Aws
|
|
126
137
|
def to_h
|
127
138
|
h = {}
|
128
139
|
instance_variables.each do |v|
|
129
|
-
v_sym = v.to_s.
|
140
|
+
v_sym = v.to_s.delete('@').to_sym
|
130
141
|
val = instance_variable_get(v)
|
131
142
|
h[v_sym] = val
|
132
143
|
end
|
@@ -13,14 +13,15 @@ module Aws
|
|
13
13
|
auto_terminate: true,
|
14
14
|
idletime: 60, # 1 minute
|
15
15
|
fallback_policy: :caller_runs # slow down the producer thread
|
16
|
+
# TODO: Consider catching the exception and sleeping instead of using :caller_runs
|
16
17
|
}.freeze
|
17
18
|
|
18
19
|
def initialize(options = {})
|
19
20
|
@executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
|
21
|
+
@retry_standard_errors = options[:retry_standard_errors]
|
20
22
|
@logger = options[:logger] || ActiveSupport::Logger.new($stdout)
|
21
23
|
end
|
22
24
|
|
23
|
-
# TODO: Consider catching the exception and sleeping instead of using :caller_runs
|
24
25
|
def execute(message)
|
25
26
|
@executor.post(message) do |message|
|
26
27
|
begin
|
@@ -31,10 +32,18 @@ module Aws
|
|
31
32
|
rescue Aws::Json::ParseError => e
|
32
33
|
@logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
|
33
34
|
rescue StandardError => e
|
34
|
-
# message will not be deleted and will be retried
|
35
35
|
job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
|
36
36
|
@logger.info "Error processing job #{job_msg}: #{e}"
|
37
37
|
@logger.debug e.backtrace.join("\n")
|
38
|
+
|
39
|
+
if @retry_standard_errors && !job.exception_executions?
|
40
|
+
@logger.info(
|
41
|
+
'retry_standard_errors is enabled and job has not ' \
|
42
|
+
"been retried by Rails. Leaving #{job_msg} in the queue."
|
43
|
+
)
|
44
|
+
else
|
45
|
+
message.delete
|
46
|
+
end
|
38
47
|
end
|
39
48
|
end
|
40
49
|
end
|
@@ -16,7 +16,8 @@ module Aws
|
|
16
16
|
threads: 2 * Concurrent.processor_count,
|
17
17
|
max_messages: 10,
|
18
18
|
shutdown_timeout: 15,
|
19
|
-
backpressure: 10
|
19
|
+
backpressure: 10,
|
20
|
+
retry_standard_errors: true
|
20
21
|
}.freeze
|
21
22
|
|
22
23
|
def initialize(args = ARGV)
|
@@ -45,7 +46,12 @@ module Aws
|
|
45
46
|
|
46
47
|
Signal.trap('INT') { raise Interrupt }
|
47
48
|
Signal.trap('TERM') { raise Interrupt }
|
48
|
-
@executor = Executor.new(
|
49
|
+
@executor = Executor.new(
|
50
|
+
max_threads: @options[:threads],
|
51
|
+
logger: @logger,
|
52
|
+
max_queue: @options[:backpressure],
|
53
|
+
retry_standard_errors: @options[:retry_standard_errors]
|
54
|
+
)
|
49
55
|
|
50
56
|
poll
|
51
57
|
rescue Interrupt
|
@@ -99,6 +105,7 @@ module Aws
|
|
99
105
|
require File.expand_path('config/environment.rb')
|
100
106
|
end
|
101
107
|
|
108
|
+
# rubocop:disable Metrics
|
102
109
|
def parse_args(argv)
|
103
110
|
out = {}
|
104
111
|
parser = ::OptionParser.new do |opts|
|
@@ -127,6 +134,10 @@ module Aws
|
|
127
134
|
'The amount of time to wait for a clean shutdown. Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
|
128
135
|
out[:shutdown_timeout] = a
|
129
136
|
end
|
137
|
+
opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
|
138
|
+
'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
|
139
|
+
out[:retry_standard_errors] = a.nil? ? true : a
|
140
|
+
end
|
130
141
|
end
|
131
142
|
|
132
143
|
parser.banner = 'aws_sqs_active_job [options]'
|
@@ -138,6 +149,7 @@ module Aws
|
|
138
149
|
parser.parse(argv)
|
139
150
|
out
|
140
151
|
end
|
152
|
+
# rubocop:enable Metrics
|
141
153
|
|
142
154
|
def validate_config
|
143
155
|
raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
|
data/lib/aws-sdk-rails.rb
CHANGED
@@ -13,6 +13,7 @@ require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
|
|
13
13
|
|
14
14
|
require_relative 'action_dispatch/session/dynamodb_store'
|
15
15
|
require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
|
16
|
+
require_relative 'active_job/queue_adapters/amazon_sqs_adapter/params'
|
16
17
|
require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
|
17
18
|
|
18
19
|
require_relative 'generators/aws_record/base'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws-sdk-rails
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-record
|
@@ -152,6 +152,7 @@ files:
|
|
152
152
|
- bin/aws_sqs_active_job
|
153
153
|
- lib/action_dispatch/session/dynamodb_store.rb
|
154
154
|
- lib/active_job/queue_adapters/amazon_sqs_adapter.rb
|
155
|
+
- lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb
|
155
156
|
- lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb
|
156
157
|
- lib/aws-sdk-rails.rb
|
157
158
|
- lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb
|
@@ -197,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
197
198
|
- !ruby/object:Gem::Version
|
198
199
|
version: '0'
|
199
200
|
requirements: []
|
200
|
-
rubygems_version: 3.4.
|
201
|
+
rubygems_version: 3.4.1
|
201
202
|
signing_key:
|
202
203
|
specification_version: 4
|
203
204
|
summary: AWS SDK for Ruby on Rails Plugin
|