aws-sdk-rails 3.9.1 → 3.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1491284a363942f9a0d4fe8712449caf2989869377fdb283632bec9c11b71b0b
4
- data.tar.gz: c1f65e37b61266b80a20c06c21a39f84f7283ade2b9a590e5b80d3963ada71c7
3
+ metadata.gz: b6ff9a7f26db95b0ae83526c594bb985b991da455979daf7d2a6ea92be0a5187
4
+ data.tar.gz: bbb91361034021720e0f43058777b530212d6c43d489f23999b636dd7e187edb
5
5
  SHA512:
6
- metadata.gz: 456ead7237daf5d83f55f4858f11d40a3d213a3bd8f8ded27c56be10e94e7aba6708a2adad8739b324991f28d22b97e32b8272d073f8f0bc082d390abfff8267
7
- data.tar.gz: 97da8e92ec42c8404af945049feae80044aa2d6d1fed276eaf40bd50baa2b07826cfd201ed7eda00a92a9d1636749aaec0cc1adbc159a300c1d5b9690c86fc96
6
+ metadata.gz: a28a3bf351332f237ef4ce23bbcdb6115ac41468d14dd3d2e1eb76c7a395c33697c9f8c55412fdad3a2b4fc63cbf7dca350243bdf5651743a7ccadfb9adebb0a
7
+ data.tar.gz: 80076e9047b439cdbd4fe8fe2da651b3cc9d23ed3346fb22c32724b9b3e00463dd77a52f99934bfe26c79ce6d9cc9a17d74fdf6683c9dc5b85e52c40a80f0d9d
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.9.1
1
+ 3.11.0
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveJob
4
+ module QueueAdapters
5
+ class AmazonSqsAdapter
6
+ # == build request parameter of Aws::SQS::Client
7
+ class Params
8
+ class << self
9
+ def assured_delay_seconds(timestamp)
10
+ delay = (timestamp - Time.now.to_f).floor
11
+ delay = 0 if delay.negative?
12
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
13
+
14
+ delay
15
+ end
16
+ end
17
+
18
+ def initialize(job, body)
19
+ @job = job
20
+ @body = body || job.serialize
21
+ end
22
+
23
+ def queue_url
24
+ @queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
25
+ end
26
+
27
+ def entry
28
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
29
+ default_entry.merge(options_for_fifo)
30
+ else
31
+ default_entry
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def default_entry
38
+ {
39
+ message_body: Aws::Json.dump(@body),
40
+ message_attributes: message_attributes
41
+ }
42
+ end
43
+
44
+ def message_attributes
45
+ {
46
+ 'aws_sqs_active_job_class' => {
47
+ string_value: @job.class.to_s,
48
+ data_type: 'String'
49
+ },
50
+ 'aws_sqs_active_job_version' => {
51
+ string_value: Aws::Rails::VERSION,
52
+ data_type: 'String'
53
+ }
54
+ }
55
+ end
56
+
57
+ def options_for_fifo
58
+ options = {}
59
+ options[:message_deduplication_id] =
60
+ Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
61
+
62
+ message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
63
+ message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
64
+
65
+ options[:message_group_id] = message_group_id
66
+ options
67
+ end
68
+
69
+ def deduplication_body
70
+ ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
71
+ ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
72
+
73
+ @body.except(*ex_dedup_keys)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -10,55 +10,44 @@ module ActiveJob
10
10
  end
11
11
 
12
12
  def enqueue_at(job, timestamp)
13
- delay = (timestamp - Time.now.to_f).floor
14
-
15
- delay = 0 if delay.negative?
16
- raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
-
13
+ delay = Params.assured_delay_seconds(timestamp)
18
14
  _enqueue(job, nil, delay_seconds: delay)
19
15
  end
20
16
 
17
+ def enqueue_all(jobs)
18
+ enqueued_count = 0
19
+ jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
20
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
21
+ base_send_message_opts = { queue_url: queue_url }
22
+
23
+ same_queue_jobs.each_slice(10) do |chunk|
24
+ entries = chunk.map do |job|
25
+ entry = Params.new(job, nil).entry
26
+ entry[:id] = job.job_id
27
+ entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
28
+ entry
29
+ end
30
+
31
+ send_message_opts = base_send_message_opts.deep_dup
32
+ send_message_opts[:entries] = entries
33
+
34
+ send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
35
+ enqueued_count += send_message_batch_result.successful.count
36
+ end
37
+ end
38
+ enqueued_count
39
+ end
40
+
21
41
  private
22
42
 
23
43
  def _enqueue(job, body = nil, send_message_opts = {})
24
44
  body ||= job.serialize
25
- queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
26
- send_message_opts[:queue_url] = queue_url
27
- send_message_opts[:message_body] = Aws::Json.dump(body)
28
- send_message_opts[:message_attributes] = message_attributes(job)
29
-
30
- if Aws::Rails::SqsActiveJob.fifo?(queue_url)
31
- send_message_opts[:message_deduplication_id] =
32
- Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body(job, body)))
33
-
34
- message_group_id = job.message_group_id if job.respond_to?(:message_group_id)
35
- message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
36
-
37
- send_message_opts[:message_group_id] = message_group_id
38
- end
45
+ params = Params.new(job, body)
46
+ send_message_opts = send_message_opts.merge(params.entry)
47
+ send_message_opts[:queue_url] = params.queue_url
39
48
 
40
49
  Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
41
50
  end
42
-
43
- def message_attributes(job)
44
- {
45
- 'aws_sqs_active_job_class' => {
46
- string_value: job.class.to_s,
47
- data_type: 'String'
48
- },
49
- 'aws_sqs_active_job_version' => {
50
- string_value: Aws::Rails::VERSION,
51
- data_type: 'String'
52
- }
53
- }
54
- end
55
-
56
- def deduplication_body(job, body)
57
- ex_dedup_keys = job.excluded_deduplication_keys if job.respond_to?(:excluded_deduplication_keys)
58
- ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
59
-
60
- body.except(*ex_dedup_keys)
61
- end
62
51
  end
63
52
 
64
53
  # create an alias to allow `:amazon` to be used as the adapter name
@@ -25,6 +25,7 @@ module Aws
25
25
  DEFAULTS = {
26
26
  max_messages: 10,
27
27
  shutdown_timeout: 15,
28
+ retry_standard_errors: true, # TODO: Remove in next MV
28
29
  queues: {},
29
30
  logger: ::Rails.logger,
30
31
  message_group_id: 'SqsActiveJobGroup',
@@ -64,6 +65,16 @@ module Aws
64
65
  # will not be deleted from the SQS queue and will be retryable after
65
66
  # the visibility timeout.
66
67
  #
68
+ # @ option options [Boolean] :retry_standard_errors
69
+ # If `true`, StandardErrors raised by ActiveJobs are left on the queue
70
+ # and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
71
+ # This behavior overrides the standard Rails ActiveJob
72
+ # [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
73
+ # behavior. When set to `true` the retries provided by this will be
74
+ # on top of any retries configured on the job with `retry_on`.
75
+ # When `false`, retry behavior is fully configured
76
+ # through `retry_on`/`discard_on` on the ActiveJobs.
77
+ #
67
78
  # @option options [ActiveSupport::Logger] :logger Logger to use
68
79
  # for the poller.
69
80
  #
@@ -126,7 +137,7 @@ module Aws
126
137
  def to_h
127
138
  h = {}
128
139
  instance_variables.each do |v|
129
- v_sym = v.to_s.gsub('@', '').to_sym
140
+ v_sym = v.to_s.delete('@').to_sym
130
141
  val = instance_variable_get(v)
131
142
  h[v_sym] = val
132
143
  end
@@ -13,14 +13,15 @@ module Aws
13
13
  auto_terminate: true,
14
14
  idletime: 60, # 1 minute
15
15
  fallback_policy: :caller_runs # slow down the producer thread
16
+ # TODO: Consider catching the exception and sleeping instead of using :caller_runs
16
17
  }.freeze
17
18
 
18
19
  def initialize(options = {})
19
20
  @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
21
+ @retry_standard_errors = options[:retry_standard_errors]
20
22
  @logger = options[:logger] || ActiveSupport::Logger.new($stdout)
21
23
  end
22
24
 
23
- # TODO: Consider catching the exception and sleeping instead of using :caller_runs
24
25
  def execute(message)
25
26
  @executor.post(message) do |message|
26
27
  begin
@@ -31,10 +32,18 @@ module Aws
31
32
  rescue Aws::Json::ParseError => e
32
33
  @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
33
34
  rescue StandardError => e
34
- # message will not be deleted and will be retried
35
35
  job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
36
36
  @logger.info "Error processing job #{job_msg}: #{e}"
37
37
  @logger.debug e.backtrace.join("\n")
38
+
39
+ if @retry_standard_errors && !job.exception_executions?
40
+ @logger.info(
41
+ 'retry_standard_errors is enabled and job has not ' \
42
+ "been retried by Rails. Leaving #{job_msg} in the queue."
43
+ )
44
+ else
45
+ message.delete
46
+ end
38
47
  end
39
48
  end
40
49
  end
@@ -15,6 +15,11 @@ module Aws
15
15
  def run
16
16
  ActiveJob::Base.execute @job_data
17
17
  end
18
+
19
+ def exception_executions?
20
+ @job_data['exception_executions'] &&
21
+ !@job_data['exception_executions'].empty?
22
+ end
18
23
  end
19
24
  end
20
25
  end
@@ -16,7 +16,8 @@ module Aws
16
16
  threads: 2 * Concurrent.processor_count,
17
17
  max_messages: 10,
18
18
  shutdown_timeout: 15,
19
- backpressure: 10
19
+ backpressure: 10,
20
+ retry_standard_errors: true
20
21
  }.freeze
21
22
 
22
23
  def initialize(args = ARGV)
@@ -45,7 +46,12 @@ module Aws
45
46
 
46
47
  Signal.trap('INT') { raise Interrupt }
47
48
  Signal.trap('TERM') { raise Interrupt }
48
- @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
49
+ @executor = Executor.new(
50
+ max_threads: @options[:threads],
51
+ logger: @logger,
52
+ max_queue: @options[:backpressure],
53
+ retry_standard_errors: @options[:retry_standard_errors]
54
+ )
49
55
 
50
56
  poll
51
57
  rescue Interrupt
@@ -99,6 +105,7 @@ module Aws
99
105
  require File.expand_path('config/environment.rb')
100
106
  end
101
107
 
108
+ # rubocop:disable Metrics
102
109
  def parse_args(argv)
103
110
  out = {}
104
111
  parser = ::OptionParser.new do |opts|
@@ -127,6 +134,10 @@ module Aws
127
134
  'The amount of time to wait for a clean shutdown. Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
128
135
  out[:shutdown_timeout] = a
129
136
  end
137
+ opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
138
+ 'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
139
+ out[:retry_standard_errors] = a.nil? ? true : a
140
+ end
130
141
  end
131
142
 
132
143
  parser.banner = 'aws_sqs_active_job [options]'
@@ -138,6 +149,7 @@ module Aws
138
149
  parser.parse(argv)
139
150
  out
140
151
  end
152
+ # rubocop:enable Metrics
141
153
 
142
154
  def validate_config
143
155
  raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
data/lib/aws-sdk-rails.rb CHANGED
@@ -13,6 +13,7 @@ require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
13
13
 
14
14
  require_relative 'action_dispatch/session/dynamodb_store'
15
15
  require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
16
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter/params'
16
17
  require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
17
18
 
18
19
  require_relative 'generators/aws_record/base'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aws-sdk-rails
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.9.1
4
+ version: 3.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-12-19 00:00:00.000000000 Z
11
+ date: 2024-03-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-record
@@ -152,6 +152,7 @@ files:
152
152
  - bin/aws_sqs_active_job
153
153
  - lib/action_dispatch/session/dynamodb_store.rb
154
154
  - lib/active_job/queue_adapters/amazon_sqs_adapter.rb
155
+ - lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb
155
156
  - lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb
156
157
  - lib/aws-sdk-rails.rb
157
158
  - lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb
@@ -197,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
197
198
  - !ruby/object:Gem::Version
198
199
  version: '0'
199
200
  requirements: []
200
- rubygems_version: 3.4.22
201
+ rubygems_version: 3.4.1
201
202
  signing_key:
202
203
  specification_version: 4
203
204
  summary: AWS SDK for Ruby on Rails Plugin