aws-sdk-rails 3.6.1 → 3.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/bin/aws_sqs_active_job +1 -0
  4. data/lib/action_dispatch/session/dynamodb_store.rb +9 -3
  5. data/lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb +78 -0
  6. data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +33 -37
  7. data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb +12 -11
  8. data/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb +31 -5
  9. data/lib/aws/rails/notifications.rb +1 -4
  10. data/lib/aws/rails/railtie.rb +9 -4
  11. data/lib/aws/rails/{mailer.rb → ses_mailer.rb} +12 -10
  12. data/lib/aws/rails/sesv2_mailer.rb +60 -0
  13. data/lib/aws/rails/sqs_active_job/configuration.rb +61 -24
  14. data/lib/aws/rails/sqs_active_job/deduplication.rb +21 -0
  15. data/lib/aws/rails/sqs_active_job/executor.rb +47 -28
  16. data/lib/aws/rails/sqs_active_job/job_runner.rb +5 -1
  17. data/lib/aws/rails/sqs_active_job/lambda_handler.rb +3 -6
  18. data/lib/aws/rails/sqs_active_job/poller.rb +56 -32
  19. data/lib/aws-sdk-rails.rb +4 -1
  20. data/lib/generators/aws_record/base.rb +164 -168
  21. data/lib/generators/aws_record/generated_attribute.rb +50 -41
  22. data/lib/generators/aws_record/model/model_generator.rb +8 -4
  23. data/lib/generators/aws_record/secondary_index.rb +31 -25
  24. data/lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb +3 -1
  25. data/lib/tasks/aws_record/migrate.rake +2 -0
  26. data/lib/tasks/dynamo_db/session_store.rake +2 -0
  27. metadata +52 -18
  28. /data/lib/generators/aws_record/model/templates/{model.rb → model.erb} +0 -0
  29. /data/lib/generators/aws_record/model/templates/{table_config.rb → table_config.erb} +0 -0
  30. /data/lib/generators/dynamo_db/session_store_migration/templates/{session_store_migration.rb → session_store_migration.erb} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 839c723ba30e8acd11e135351278a0422b4f7e6a09a364b8a3fda7d33aa2e899
4
- data.tar.gz: af1ba6b6525f17e6a46cc16aa4ccbaaf8317860ddf5cd6128d3883c82660ee24
3
+ metadata.gz: 43528604142d01e2ddcddf37ae598a08c11744aa3a9ccc6757dbc7d1932d3546
4
+ data.tar.gz: 7977e087f6372fc8326193dad744f908cda1d2174fc6f817e9cfe48b286a5cfb
5
5
  SHA512:
6
- metadata.gz: 8288a0c8551bc30cac782c7510a825fdcd9d8ba31bd1d2feda8c2a20ad29f569194237f686f191f1a57ee12eb91a08bb1c5b65b07519af5b62bd7f56ab3f79cb
7
- data.tar.gz: d6835fc12e6a4a5bc9573b4c5b997a1df5d17e49c28ea2ea16e76f8ebc7b2c8c515026b0a48fb0b15eaab3ba22f1904782f553a03a784dcd665d71723ddd47bd
6
+ metadata.gz: 2ceda136ba4f7077b6608787528f2ae97b9ceb3169d9c67f101105e7e512dc8000cdb3bf09e5f96b786544bdb427eb15cc0df6571ecc544f5b1603c51cc034b8
7
+ data.tar.gz: 1fdb53b5fdc4a752643deb9e360988983ad939cce6ada4a8ce3bc68122959c44c7ca6f98d55987172c4d559d9d010ec652b482576bf12b4e496193b183aa236e
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.6.1
1
+ 3.13.0
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
4
  require_relative '../lib/aws/rails/sqs_active_job/poller'
4
5
 
@@ -1,4 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'aws-sessionstore-dynamodb'
4
+ require 'action_dispatch/middleware/session/abstract_store'
2
5
 
3
6
  module ActionDispatch
4
7
  module Session
@@ -9,13 +12,16 @@ module ActionDispatch
9
12
  # This class will use the Rails secret_key_base unless otherwise provided.
10
13
  #
11
14
  # Configuration can also be provided in YAML files from Rails config, either
12
- # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
15
+ # in "config/session_store.yml" or "config/session_store/#\\{Rails.env}.yml".
13
16
  # Configuration files that are environment-specific will take precedence.
14
17
  #
15
18
  # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
16
19
  class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
20
+ include StaleSessionCheck
21
+ include SessionObject
22
+
17
23
  def initialize(app, options = {})
18
- options[:config_file] ||= config_file if config_file.exist?
24
+ options[:config_file] ||= config_file if File.exist?(config_file)
19
25
  options[:secret_key] ||= Rails.application.secret_key_base
20
26
  super
21
27
  end
@@ -24,7 +30,7 @@ module ActionDispatch
24
30
 
25
31
  def config_file
26
32
  file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
27
- file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
33
+ file = Rails.root.join('config/dynamo_db_session_store.yml') unless File.exist?(file)
28
34
  file
29
35
  end
30
36
  end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveJob
4
+ module QueueAdapters
5
+ class AmazonSqsAdapter
6
+ # == build request parameter of Aws::SQS::Client
7
+ class Params
8
+ class << self
9
+ def assured_delay_seconds(timestamp)
10
+ delay = (timestamp - Time.now.to_f).floor
11
+ delay = 0 if delay.negative?
12
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
13
+
14
+ delay
15
+ end
16
+ end
17
+
18
+ def initialize(job, body)
19
+ @job = job
20
+ @body = body || job.serialize
21
+ end
22
+
23
+ def queue_url
24
+ @queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
25
+ end
26
+
27
+ def entry
28
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
29
+ default_entry.merge(options_for_fifo)
30
+ else
31
+ default_entry
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def default_entry
38
+ {
39
+ message_body: Aws::Json.dump(@body),
40
+ message_attributes: message_attributes
41
+ }
42
+ end
43
+
44
+ def message_attributes
45
+ {
46
+ 'aws_sqs_active_job_class' => {
47
+ string_value: @job.class.to_s,
48
+ data_type: 'String'
49
+ },
50
+ 'aws_sqs_active_job_version' => {
51
+ string_value: Aws::Rails::VERSION,
52
+ data_type: 'String'
53
+ }
54
+ }
55
+ end
56
+
57
+ def options_for_fifo
58
+ options = {}
59
+ options[:message_deduplication_id] =
60
+ Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
61
+
62
+ message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
63
+ message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
64
+
65
+ options[:message_group_id] = message_group_id
66
+ options
67
+ end
68
+
69
+ def deduplication_body
70
+ ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
71
+ ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
72
+
73
+ @body.except(*ex_dedup_keys)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -4,53 +4,49 @@ require 'aws-sdk-sqs'
4
4
 
5
5
  module ActiveJob
6
6
  module QueueAdapters
7
-
8
7
  class AmazonSqsAdapter
9
-
10
8
  def enqueue(job)
11
9
  _enqueue(job)
12
10
  end
13
11
 
14
- def enqueue_at(job, timestamp, opts={})
15
- delay = (timestamp - Time.now.to_f).floor
16
- raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
- _enqueue(job, delay_seconds: delay)
12
+ def enqueue_at(job, timestamp)
13
+ delay = Params.assured_delay_seconds(timestamp)
14
+ _enqueue(job, nil, delay_seconds: delay)
18
15
  end
19
16
 
20
- private
21
-
22
- def _enqueue(job, send_message_opts = {})
23
- body = job.serialize
24
- queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
- send_message_opts[:queue_url] = queue_url
26
- send_message_opts[:message_body] = Aws::Json.dump(body)
27
- send_message_opts[:message_attributes] = message_attributes(job)
28
-
29
- if Aws::Rails::SqsActiveJob.fifo?(queue_url)
30
- # job_id is unique per initialization of job
31
- # Remove it from message dup id to ensure run-once behavior
32
- # with ActiveJob retries
33
- send_message_opts[:message_deduplication_id] =
34
- Digest::SHA256.hexdigest(
35
- Aws::Json.dump(body.except('job_id'))
36
- )
37
-
38
- send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
17
+ def enqueue_all(jobs)
18
+ enqueued_count = 0
19
+ jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
20
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
21
+ base_send_message_opts = { queue_url: queue_url }
22
+
23
+ same_queue_jobs.each_slice(10) do |chunk|
24
+ entries = chunk.map do |job|
25
+ entry = Params.new(job, nil).entry
26
+ entry[:id] = job.job_id
27
+ entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
28
+ entry
29
+ end
30
+
31
+ send_message_opts = base_send_message_opts.deep_dup
32
+ send_message_opts[:entries] = entries
33
+
34
+ send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
35
+ enqueued_count += send_message_batch_result.successful.count
36
+ end
39
37
  end
40
- Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
38
+ enqueued_count
41
39
  end
42
40
 
43
- def message_attributes(job)
44
- {
45
- 'aws_sqs_active_job_class' => {
46
- string_value: job.class.to_s,
47
- data_type: 'String'
48
- },
49
- 'aws_sqs_active_job_version' => {
50
- string_value: Aws::Rails::VERSION,
51
- data_type: 'String'
52
- }
53
- }
41
+ private
42
+
43
+ def _enqueue(job, body = nil, send_message_opts = {})
44
+ body ||= job.serialize
45
+ params = Params.new(job, body)
46
+ send_message_opts = send_message_opts.merge(params.entry)
47
+ send_message_opts[:queue_url] = params.queue_url
48
+
49
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
54
50
  end
55
51
  end
56
52
 
@@ -5,7 +5,6 @@ require 'concurrent'
5
5
 
6
6
  module ActiveJob
7
7
  module QueueAdapters
8
-
9
8
  # == Async adapter for Amazon SQS ActiveJob
10
9
  #
11
10
  # This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
@@ -15,22 +14,24 @@ module ActiveJob
15
14
  #
16
15
  # config.active_job.queue_adapter = :amazon_sqs_async
17
16
  class AmazonSqsAsyncAdapter < AmazonSqsAdapter
18
-
19
17
  private
20
18
 
21
- def _enqueue(job, send_message_opts = {})
19
+ def _enqueue(job, body = nil, send_message_opts = {})
22
20
  # FIFO jobs must be queued in order, so do not queue async
23
21
  queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
24
22
  if Aws::Rails::SqsActiveJob.fifo?(queue_url)
25
- super(job, send_message_opts)
23
+ super
26
24
  else
27
- Concurrent::Promise
28
- .execute { super(job, send_message_opts) }
29
- .on_error do |e|
30
- Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
31
- error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
32
- error_handler.call(e, job, send_message_opts) if error_handler
33
- end
25
+ # Serialize is called here because the job’s locale needs to be
26
+ # determined in this thread and not in some other thread.
27
+ body = job.serialize
28
+ Concurrent::Promises
29
+ .future { super }
30
+ .rescue do |e|
31
+ Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
32
+ error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
33
+ error_handler&.call(e, job, send_message_opts)
34
+ end
34
35
  end
35
36
  end
36
37
  end
@@ -11,7 +11,7 @@ module Aws
11
11
 
12
12
  def initialize(app)
13
13
  @app = app
14
- @logger = ActiveSupport::Logger.new(STDOUT)
14
+ @logger = ::Rails.logger
15
15
  end
16
16
 
17
17
  def call(env)
@@ -20,11 +20,11 @@ module Aws
20
20
  # Pass through unless user agent is the SQS Daemon
21
21
  return @app.call(env) unless from_sqs_daemon?(request)
22
22
 
23
- @logger.debug('aws-rails-sdk middleware detected call from Elastic Beanstalk SQS Daemon.')
23
+ @logger.debug('aws-sdk-rails middleware detected call from Elastic Beanstalk SQS Daemon.')
24
24
 
25
25
  # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
26
26
  unless request.local? || sent_from_docker_host?(request)
27
- @logger.warn("SQSD request detected from untrusted address #{request.remote_ip}; returning 403 forbidden.")
27
+ @logger.warn("SQSD request detected from untrusted address #{request.ip}; returning 403 forbidden.")
28
28
  return FORBIDDEN_RESPONSE
29
29
  end
30
30
 
@@ -81,11 +81,37 @@ module Aws
81
81
  end
82
82
 
83
83
  def sent_from_docker_host?(request)
84
- app_runs_in_docker_container? && request.remote_ip == '172.17.0.1'
84
+ app_runs_in_docker_container? && default_gw_ips.include?(request.ip)
85
85
  end
86
86
 
87
87
  def app_runs_in_docker_container?
88
- @app_runs_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
88
+ @app_runs_in_docker_container ||= in_docker_container_with_cgroup1? || in_docker_container_with_cgroup2?
89
+ end
90
+
91
+ def in_docker_container_with_cgroup1?
92
+ File.exist?('/proc/1/cgroup') && File.read('/proc/1/cgroup') =~ %r{/docker/}
93
+ end
94
+
95
+ def in_docker_container_with_cgroup2?
96
+ File.exist?('/proc/self/mountinfo') && File.read('/proc/self/mountinfo') =~ %r{/docker/containers/}
97
+ end
98
+
99
+ def default_gw_ips
100
+ default_gw_ips = ['172.17.0.1']
101
+
102
+ if File.exist?('/proc/net/route')
103
+ File.open('/proc/net/route').each_line do |line|
104
+ fields = line.strip.split
105
+ next if fields.size != 11
106
+
107
+ # Destination == 0.0.0.0 and Flags & RTF_GATEWAY != 0
108
+ if fields[1] == '00000000' && (fields[3].hex & 0x2) != 0
109
+ default_gw_ips << IPAddr.new_ntoh([fields[2].hex].pack('L')).to_s
110
+ end
111
+ end
112
+ end
113
+
114
+ default_gw_ips
89
115
  end
90
116
  end
91
117
  end
@@ -5,14 +5,12 @@ require 'active_support/notifications'
5
5
 
6
6
  module Aws
7
7
  module Rails
8
-
9
8
  # Instruments client operation calls for ActiveSupport::Notifications
10
9
  # Each client operation will produce an event with name:
11
10
  # <operation>.<service>.aws
12
11
  # @api private
13
12
  class Notifications < Seahorse::Client::Plugin
14
-
15
- def add_handlers(handlers, config)
13
+ def add_handlers(handlers, _config)
16
14
  # This plugin needs to be first
17
15
  # which means it is called first in the stack, to start recording time,
18
16
  # and returns last
@@ -20,7 +18,6 @@ module Aws
20
18
  end
21
19
 
22
20
  class Handler < Seahorse::Client::Handler
23
-
24
21
  def call(context)
25
22
  event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
26
23
  ActiveSupport::Notifications.instrument(event_name, context: context) do
@@ -10,6 +10,7 @@ module Aws
10
10
  # Initialization Actions
11
11
  Aws::Rails.use_rails_encrypted_credentials
12
12
  Aws::Rails.add_action_mailer_delivery_method
13
+ Aws::Rails.add_action_mailer_delivery_method(:sesv2)
13
14
  Aws::Rails.log_to_rails_logger
14
15
  end
15
16
 
@@ -28,11 +29,15 @@ module Aws
28
29
  #
29
30
  # @param [Symbol] name The name of the ActionMailer delivery method to
30
31
  # register.
31
- # @param [Hash] options The options you wish to pass on to the
32
- # Aws::SES::Client initialization method.
33
- def self.add_action_mailer_delivery_method(name = :ses, options = {})
32
+ # @param [Hash] client_options The options you wish to pass on to the
33
+ # Aws::SES[V2]::Client initialization method.
34
+ def self.add_action_mailer_delivery_method(name = :ses, client_options = {})
34
35
  ActiveSupport.on_load(:action_mailer) do
35
- add_delivery_method(name, Aws::Rails::Mailer, options)
36
+ if name == :sesv2
37
+ add_delivery_method(name, Aws::Rails::Sesv2Mailer, client_options)
38
+ else
39
+ add_delivery_method(name, Aws::Rails::SesMailer, client_options)
40
+ end
36
41
  end
37
42
  end
38
43
 
@@ -15,25 +15,23 @@ module Aws
15
15
  #
16
16
  # Uses the AWS SDK for Ruby's credential provider chain when creating an SES
17
17
  # client instance.
18
- class Mailer
18
+ class SesMailer
19
19
  # @param [Hash] options Passes along initialization options to
20
20
  # [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
21
21
  def initialize(options = {})
22
22
  @client = SES::Client.new(options)
23
+ @client.config.user_agent_frameworks << 'aws-sdk-rails'
23
24
  end
24
25
 
25
26
  # Rails expects this method to exist, and to handle a Mail::Message object
26
27
  # correctly. Called during mail delivery.
27
28
  def deliver!(message)
28
- send_opts = {}
29
- send_opts[:raw_message] = {}
30
- send_opts[:raw_message][:data] = message.to_s
31
-
32
- if message.respond_to?(:destinations)
33
- send_opts[:destinations] = message.destinations
34
- end
35
-
36
- @client.send_raw_email(send_opts).tap do |response|
29
+ params = {
30
+ raw_message: { data: message.to_s },
31
+ source: message.smtp_envelope_from, # defaults to From header
32
+ destinations: message.smtp_envelope_to # defaults to destinations (To,Cc,Bcc)
33
+ }
34
+ @client.send_raw_email(params).tap do |response|
37
35
  message.header[:ses_message_id] = response.message_id
38
36
  end
39
37
  end
@@ -45,3 +43,7 @@ module Aws
45
43
  end
46
44
  end
47
45
  end
46
+
47
+ # This is for backwards compatibility after introducing support for SESv2.
48
+ # The old mailer is now replaced with the new SES (v1) mailer.
49
+ Aws::Rails::Mailer = Aws::Rails::SesMailer
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sesv2'
4
+
5
+ module Aws
6
+ module Rails
7
+ # Provides a delivery method for ActionMailer that uses Amazon Simple Email
8
+ # Service V2.
9
+ #
10
+ # Once you have an SESv2 delivery method you can configure Rails to
11
+ # use this for ActionMailer in your environment configuration
12
+ # (e.g. RAILS_ROOT/config/environments/production.rb)
13
+ #
14
+ # config.action_mailer.delivery_method = :sesv2
15
+ #
16
+ # Uses the AWS SDK for Ruby's credential provider chain when creating an SESV2
17
+ # client instance.
18
+ class Sesv2Mailer
19
+ # @param [Hash] options Passes along initialization options to
20
+ # [Aws::SESV2::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SESV2/Client.html#initialize-instance_method).
21
+ def initialize(options = {})
22
+ @client = SESV2::Client.new(options)
23
+ @client.config.user_agent_frameworks << 'aws-sdk-rails'
24
+ end
25
+
26
+ # Rails expects this method to exist, and to handle a Mail::Message object
27
+ # correctly. Called during mail delivery.
28
+ def deliver!(message)
29
+ params = { content: { raw: { data: message.to_s } } }
30
+ # smtp_envelope_from will default to the From address *without* sender names.
31
+ # By omitting this param, SESv2 will correctly use sender names from the mail headers.
32
+ # We should only use smtp_envelope_from when it was explicitly set (instance variable set)
33
+ params[:from_email_address] = message.smtp_envelope_from if message.instance_variable_get(:@smtp_envelope_from)
34
+ params[:destination] = {
35
+ to_addresses: to_addresses(message),
36
+ cc_addresses: message.cc,
37
+ bcc_addresses: message.bcc
38
+ }
39
+
40
+ @client.send_email(params).tap do |response|
41
+ message.header[:ses_message_id] = response.message_id
42
+ end
43
+ end
44
+
45
+ # ActionMailer expects this method to be present and to return a hash.
46
+ def settings
47
+ {}
48
+ end
49
+
50
+ private
51
+
52
+ # smtp_envelope_to will default to the full destinations (To, Cc, Bcc)
53
+ # SES v2 API prefers each component split out into a destination hash.
54
+ # When smtp_envelope_to was set, use it explicitly for to_address only.
55
+ def to_addresses(message)
56
+ message.instance_variable_get(:@smtp_envelope_to) ? message.smtp_envelope_to : message.to
57
+ end
58
+ end
59
+ end
60
+ end
@@ -3,7 +3,6 @@
3
3
  module Aws
4
4
  module Rails
5
5
  module SqsActiveJob
6
-
7
6
  # @return [Configuration] the (singleton) Configuration
8
7
  def self.config
9
8
  @config ||= Configuration.new
@@ -21,24 +20,26 @@ module Aws
21
20
  # Configuration for AWS SQS ActiveJob.
22
21
  # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
23
22
  class Configuration
24
-
25
23
  # Default configuration options
26
24
  # @api private
27
25
  DEFAULTS = {
28
- max_messages: 10,
29
- visibility_timeout: 120,
26
+ max_messages: 10,
30
27
  shutdown_timeout: 15,
28
+ retry_standard_errors: true, # TODO: Remove in next MV
31
29
  queues: {},
32
30
  logger: ::Rails.logger,
33
- message_group_id: 'SqsActiveJobGroup'
34
- }
31
+ message_group_id: 'SqsActiveJobGroup',
32
+ excluded_deduplication_keys: ['job_id']
33
+ }.freeze
35
34
 
36
35
  # @api private
37
36
  attr_accessor :queues, :max_messages, :visibility_timeout,
38
37
  :shutdown_timeout, :client, :logger,
39
38
  :async_queue_error_handler, :message_group_id
40
39
 
41
- # Don't use this method directly: Confugration is a singleton class, use
40
+ attr_reader :excluded_deduplication_keys
41
+
42
+ # Don't use this method directly: Configuration is a singleton class, use
42
43
  # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
43
44
  #
44
45
  # @param [Hash] options
@@ -50,6 +51,8 @@ module Aws
50
51
  # The max number of messages to poll for in a batch.
51
52
  #
52
53
  # @option options [Integer] :visibility_timeout
54
+ # If unset, the visibility timeout configured on the
55
+ # SQS queue will be used.
53
56
  # The visibility timeout is the number of seconds
54
57
  # that a message will not be processable by any other consumers.
55
58
  # You should set this value to be longer than your expected job runtime
@@ -62,11 +65,21 @@ module Aws
62
65
  # will not be deleted from the SQS queue and will be retryable after
63
66
  # the visibility timeout.
64
67
  #
68
+ # @ option options [Boolean] :retry_standard_errors
69
+ # If `true`, StandardErrors raised by ActiveJobs are left on the queue
70
+ # and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
71
+ # This behavior overrides the standard Rails ActiveJob
72
+ # [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
73
+ # behavior. When set to `true` the retries provided by this will be
74
+ # on top of any retries configured on the job with `retry_on`.
75
+ # When `false`, retry behavior is fully configured
76
+ # through `retry_on`/`discard_on` on the ActiveJobs.
77
+ #
65
78
  # @option options [ActiveSupport::Logger] :logger Logger to use
66
79
  # for the poller.
67
80
  #
68
81
  # @option options [String] :config_file
69
- # Override file to load configuration from. If not specified will
82
+ # Override file to load configuration from. If not specified will
70
83
  # attempt to load from config/aws_sqs_active_job.yml.
71
84
  #
72
85
  # @option options [String] :message_group_id (SqsActiveJobGroup)
@@ -80,18 +93,31 @@ module Aws
80
93
  # +active_job.queue_adapter = :amazon_sqs_async+. Called with:
81
94
  # [error, job, job_options]
82
95
  #
83
- # @option options [SQS::Client] :client SQS Client to use. A default
96
+ # @option options [SQS::Client] :client SQS Client to use. A default
84
97
  # client will be created if none is provided.
98
+ #
99
+ # @option options [Array] :excluded_deduplication_keys (['job_id'])
100
+ # The type of keys stored in the array should be String or Symbol.
101
+ # Using this option, job_id is implicitly added to the keys.
102
+
85
103
  def initialize(options = {})
86
- options[:config_file] ||= config_file if config_file.exist?
104
+ options[:config_file] ||= config_file if File.exist?(config_file)
87
105
  options = DEFAULTS
88
- .merge(file_options(options))
89
- .merge(options)
106
+ .merge(file_options(options))
107
+ .merge(options)
90
108
  set_attributes(options)
91
109
  end
92
110
 
111
+ def excluded_deduplication_keys=(keys)
112
+ @excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
113
+ end
114
+
93
115
  def client
94
- @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
116
+ @client ||= begin
117
+ client = Aws::SQS::Client.new
118
+ client.config.user_agent_frameworks << 'aws-sdk-rails'
119
+ client
120
+ end
95
121
  end
96
122
 
97
123
  # Return the queue_url for a given job_queue name
@@ -99,7 +125,7 @@ module Aws
99
125
  job_queue = job_queue.to_sym
100
126
  raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
101
127
 
102
- queues[job_queue.to_sym]
128
+ queues[job_queue]
103
129
  end
104
130
 
105
131
  # @api private
@@ -110,9 +136,9 @@ module Aws
110
136
  # @api private
111
137
  def to_h
112
138
  h = {}
113
- self.instance_variables.each do |v|
114
- v_sym = v.to_s.gsub('@', '').to_sym
115
- val = self.instance_variable_get(v)
139
+ instance_variables.each do |v|
140
+ v_sym = v.to_s.delete('@').to_sym
141
+ val = instance_variable_get(v)
116
142
  h[v_sym] = val
117
143
  end
118
144
  h
@@ -122,8 +148,9 @@ module Aws
122
148
 
123
149
  # Set accessible attributes after merged options.
124
150
  def set_attributes(options)
125
- options.keys.each do |opt_name|
151
+ options.each_key do |opt_name|
126
152
  instance_variable_set("@#{opt_name}", options[opt_name])
153
+ client.config.user_agent_frameworks << 'aws-sdk-rails' if opt_name == :client
127
154
  end
128
155
  end
129
156
 
@@ -138,24 +165,34 @@ module Aws
138
165
 
139
166
  def config_file
140
167
  file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
141
- file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
168
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless File.exist?(file)
142
169
  file
143
170
  end
144
171
 
145
172
  # Load options from YAML file
146
173
  def load_from_file(file_path)
147
- require "erb"
148
- opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
174
+ opts = load_yaml(file_path) || {}
149
175
  opts.deep_symbolize_keys
150
176
  end
151
177
 
152
178
  # @return [String] Configuration path found in environment or YAML file.
153
179
  def config_file_path(options)
154
- options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
180
+ options[:config_file] || ENV.fetch('AWS_SQS_ACTIVE_JOB_CONFIG_FILE', nil)
155
181
  end
156
182
 
157
- def user_agent
158
- "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
183
+ def load_yaml(file_path)
184
+ require 'erb'
185
+ source = ERB.new(File.read(file_path)).result
186
+
187
+ # Avoid incompatible changes with Psych 4.0.0
188
+ # https://bugs.ruby-lang.org/issues/17866
189
+ # rubocop:disable Security/YAMLLoad
190
+ begin
191
+ YAML.load(source, aliases: true) || {}
192
+ rescue ArgumentError
193
+ YAML.load(source) || {}
194
+ end
195
+ # rubocop:enable Security/YAMLLoad
159
196
  end
160
197
  end
161
198
  end