aws-sdk-rails 3.6.1 → 3.13.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/bin/aws_sqs_active_job +1 -0
  4. data/lib/action_dispatch/session/dynamodb_store.rb +9 -3
  5. data/lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb +78 -0
  6. data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +33 -37
  7. data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb +12 -11
  8. data/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb +31 -5
  9. data/lib/aws/rails/notifications.rb +1 -4
  10. data/lib/aws/rails/railtie.rb +9 -4
  11. data/lib/aws/rails/{mailer.rb → ses_mailer.rb} +12 -10
  12. data/lib/aws/rails/sesv2_mailer.rb +60 -0
  13. data/lib/aws/rails/sqs_active_job/configuration.rb +61 -24
  14. data/lib/aws/rails/sqs_active_job/deduplication.rb +21 -0
  15. data/lib/aws/rails/sqs_active_job/executor.rb +47 -28
  16. data/lib/aws/rails/sqs_active_job/job_runner.rb +5 -1
  17. data/lib/aws/rails/sqs_active_job/lambda_handler.rb +3 -6
  18. data/lib/aws/rails/sqs_active_job/poller.rb +56 -32
  19. data/lib/aws-sdk-rails.rb +4 -1
  20. data/lib/generators/aws_record/base.rb +164 -168
  21. data/lib/generators/aws_record/generated_attribute.rb +50 -41
  22. data/lib/generators/aws_record/model/model_generator.rb +8 -4
  23. data/lib/generators/aws_record/secondary_index.rb +31 -25
  24. data/lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb +3 -1
  25. data/lib/tasks/aws_record/migrate.rake +2 -0
  26. data/lib/tasks/dynamo_db/session_store.rake +2 -0
  27. metadata +52 -18
  28. /data/lib/generators/aws_record/model/templates/{model.rb → model.erb} +0 -0
  29. /data/lib/generators/aws_record/model/templates/{table_config.rb → table_config.erb} +0 -0
  30. /data/lib/generators/dynamo_db/session_store_migration/templates/{session_store_migration.rb → session_store_migration.erb} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 839c723ba30e8acd11e135351278a0422b4f7e6a09a364b8a3fda7d33aa2e899
4
- data.tar.gz: af1ba6b6525f17e6a46cc16aa4ccbaaf8317860ddf5cd6128d3883c82660ee24
3
+ metadata.gz: 43528604142d01e2ddcddf37ae598a08c11744aa3a9ccc6757dbc7d1932d3546
4
+ data.tar.gz: 7977e087f6372fc8326193dad744f908cda1d2174fc6f817e9cfe48b286a5cfb
5
5
  SHA512:
6
- metadata.gz: 8288a0c8551bc30cac782c7510a825fdcd9d8ba31bd1d2feda8c2a20ad29f569194237f686f191f1a57ee12eb91a08bb1c5b65b07519af5b62bd7f56ab3f79cb
7
- data.tar.gz: d6835fc12e6a4a5bc9573b4c5b997a1df5d17e49c28ea2ea16e76f8ebc7b2c8c515026b0a48fb0b15eaab3ba22f1904782f553a03a784dcd665d71723ddd47bd
6
+ metadata.gz: 2ceda136ba4f7077b6608787528f2ae97b9ceb3169d9c67f101105e7e512dc8000cdb3bf09e5f96b786544bdb427eb15cc0df6571ecc544f5b1603c51cc034b8
7
+ data.tar.gz: 1fdb53b5fdc4a752643deb9e360988983ad939cce6ada4a8ce3bc68122959c44c7ca6f98d55987172c4d559d9d010ec652b482576bf12b4e496193b183aa236e
data/VERSION CHANGED
@@ -1 +1 @@
1
- 3.6.1
1
+ 3.13.0
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
4
  require_relative '../lib/aws/rails/sqs_active_job/poller'
4
5
 
@@ -1,4 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'aws-sessionstore-dynamodb'
4
+ require 'action_dispatch/middleware/session/abstract_store'
2
5
 
3
6
  module ActionDispatch
4
7
  module Session
@@ -9,13 +12,16 @@ module ActionDispatch
9
12
  # This class will use the Rails secret_key_base unless otherwise provided.
10
13
  #
11
14
  # Configuration can also be provided in YAML files from Rails config, either
12
- # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
15
+ # in "config/session_store.yml" or "config/session_store/#\\{Rails.env}.yml".
13
16
  # Configuration files that are environment-specific will take precedence.
14
17
  #
15
18
  # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
16
19
  class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
20
+ include StaleSessionCheck
21
+ include SessionObject
22
+
17
23
  def initialize(app, options = {})
18
- options[:config_file] ||= config_file if config_file.exist?
24
+ options[:config_file] ||= config_file if File.exist?(config_file)
19
25
  options[:secret_key] ||= Rails.application.secret_key_base
20
26
  super
21
27
  end
@@ -24,7 +30,7 @@ module ActionDispatch
24
30
 
25
31
  def config_file
26
32
  file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
27
- file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
33
+ file = Rails.root.join('config/dynamo_db_session_store.yml') unless File.exist?(file)
28
34
  file
29
35
  end
30
36
  end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveJob
4
+ module QueueAdapters
5
+ class AmazonSqsAdapter
6
+ # == build request parameter of Aws::SQS::Client
7
+ class Params
8
+ class << self
9
+ def assured_delay_seconds(timestamp)
10
+ delay = (timestamp - Time.now.to_f).floor
11
+ delay = 0 if delay.negative?
12
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
13
+
14
+ delay
15
+ end
16
+ end
17
+
18
+ def initialize(job, body)
19
+ @job = job
20
+ @body = body || job.serialize
21
+ end
22
+
23
+ def queue_url
24
+ @queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
25
+ end
26
+
27
+ def entry
28
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
29
+ default_entry.merge(options_for_fifo)
30
+ else
31
+ default_entry
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def default_entry
38
+ {
39
+ message_body: Aws::Json.dump(@body),
40
+ message_attributes: message_attributes
41
+ }
42
+ end
43
+
44
+ def message_attributes
45
+ {
46
+ 'aws_sqs_active_job_class' => {
47
+ string_value: @job.class.to_s,
48
+ data_type: 'String'
49
+ },
50
+ 'aws_sqs_active_job_version' => {
51
+ string_value: Aws::Rails::VERSION,
52
+ data_type: 'String'
53
+ }
54
+ }
55
+ end
56
+
57
+ def options_for_fifo
58
+ options = {}
59
+ options[:message_deduplication_id] =
60
+ Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
61
+
62
+ message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
63
+ message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
64
+
65
+ options[:message_group_id] = message_group_id
66
+ options
67
+ end
68
+
69
+ def deduplication_body
70
+ ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
71
+ ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
72
+
73
+ @body.except(*ex_dedup_keys)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -4,53 +4,49 @@ require 'aws-sdk-sqs'
4
4
 
5
5
  module ActiveJob
6
6
  module QueueAdapters
7
-
8
7
  class AmazonSqsAdapter
9
-
10
8
  def enqueue(job)
11
9
  _enqueue(job)
12
10
  end
13
11
 
14
- def enqueue_at(job, timestamp, opts={})
15
- delay = (timestamp - Time.now.to_f).floor
16
- raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
- _enqueue(job, delay_seconds: delay)
12
+ def enqueue_at(job, timestamp)
13
+ delay = Params.assured_delay_seconds(timestamp)
14
+ _enqueue(job, nil, delay_seconds: delay)
18
15
  end
19
16
 
20
- private
21
-
22
- def _enqueue(job, send_message_opts = {})
23
- body = job.serialize
24
- queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
- send_message_opts[:queue_url] = queue_url
26
- send_message_opts[:message_body] = Aws::Json.dump(body)
27
- send_message_opts[:message_attributes] = message_attributes(job)
28
-
29
- if Aws::Rails::SqsActiveJob.fifo?(queue_url)
30
- # job_id is unique per initialization of job
31
- # Remove it from message dup id to ensure run-once behavior
32
- # with ActiveJob retries
33
- send_message_opts[:message_deduplication_id] =
34
- Digest::SHA256.hexdigest(
35
- Aws::Json.dump(body.except('job_id'))
36
- )
37
-
38
- send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
17
+ def enqueue_all(jobs)
18
+ enqueued_count = 0
19
+ jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
20
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
21
+ base_send_message_opts = { queue_url: queue_url }
22
+
23
+ same_queue_jobs.each_slice(10) do |chunk|
24
+ entries = chunk.map do |job|
25
+ entry = Params.new(job, nil).entry
26
+ entry[:id] = job.job_id
27
+ entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
28
+ entry
29
+ end
30
+
31
+ send_message_opts = base_send_message_opts.deep_dup
32
+ send_message_opts[:entries] = entries
33
+
34
+ send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
35
+ enqueued_count += send_message_batch_result.successful.count
36
+ end
39
37
  end
40
- Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
38
+ enqueued_count
41
39
  end
42
40
 
43
- def message_attributes(job)
44
- {
45
- 'aws_sqs_active_job_class' => {
46
- string_value: job.class.to_s,
47
- data_type: 'String'
48
- },
49
- 'aws_sqs_active_job_version' => {
50
- string_value: Aws::Rails::VERSION,
51
- data_type: 'String'
52
- }
53
- }
41
+ private
42
+
43
+ def _enqueue(job, body = nil, send_message_opts = {})
44
+ body ||= job.serialize
45
+ params = Params.new(job, body)
46
+ send_message_opts = send_message_opts.merge(params.entry)
47
+ send_message_opts[:queue_url] = params.queue_url
48
+
49
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
54
50
  end
55
51
  end
56
52
 
@@ -5,7 +5,6 @@ require 'concurrent'
5
5
 
6
6
  module ActiveJob
7
7
  module QueueAdapters
8
-
9
8
  # == Async adapter for Amazon SQS ActiveJob
10
9
  #
11
10
  # This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
@@ -15,22 +14,24 @@ module ActiveJob
15
14
  #
16
15
  # config.active_job.queue_adapter = :amazon_sqs_async
17
16
  class AmazonSqsAsyncAdapter < AmazonSqsAdapter
18
-
19
17
  private
20
18
 
21
- def _enqueue(job, send_message_opts = {})
19
+ def _enqueue(job, body = nil, send_message_opts = {})
22
20
  # FIFO jobs must be queued in order, so do not queue async
23
21
  queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
24
22
  if Aws::Rails::SqsActiveJob.fifo?(queue_url)
25
- super(job, send_message_opts)
23
+ super
26
24
  else
27
- Concurrent::Promise
28
- .execute { super(job, send_message_opts) }
29
- .on_error do |e|
30
- Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
31
- error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
32
- error_handler.call(e, job, send_message_opts) if error_handler
33
- end
25
+ # Serialize is called here because the job’s locale needs to be
26
+ # determined in this thread and not in some other thread.
27
+ body = job.serialize
28
+ Concurrent::Promises
29
+ .future { super }
30
+ .rescue do |e|
31
+ Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
32
+ error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
33
+ error_handler&.call(e, job, send_message_opts)
34
+ end
34
35
  end
35
36
  end
36
37
  end
@@ -11,7 +11,7 @@ module Aws
11
11
 
12
12
  def initialize(app)
13
13
  @app = app
14
- @logger = ActiveSupport::Logger.new(STDOUT)
14
+ @logger = ::Rails.logger
15
15
  end
16
16
 
17
17
  def call(env)
@@ -20,11 +20,11 @@ module Aws
20
20
  # Pass through unless user agent is the SQS Daemon
21
21
  return @app.call(env) unless from_sqs_daemon?(request)
22
22
 
23
- @logger.debug('aws-rails-sdk middleware detected call from Elastic Beanstalk SQS Daemon.')
23
+ @logger.debug('aws-sdk-rails middleware detected call from Elastic Beanstalk SQS Daemon.')
24
24
 
25
25
  # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
26
26
  unless request.local? || sent_from_docker_host?(request)
27
- @logger.warn("SQSD request detected from untrusted address #{request.remote_ip}; returning 403 forbidden.")
27
+ @logger.warn("SQSD request detected from untrusted address #{request.ip}; returning 403 forbidden.")
28
28
  return FORBIDDEN_RESPONSE
29
29
  end
30
30
 
@@ -81,11 +81,37 @@ module Aws
81
81
  end
82
82
 
83
83
  def sent_from_docker_host?(request)
84
- app_runs_in_docker_container? && request.remote_ip == '172.17.0.1'
84
+ app_runs_in_docker_container? && default_gw_ips.include?(request.ip)
85
85
  end
86
86
 
87
87
  def app_runs_in_docker_container?
88
- @app_runs_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
88
+ @app_runs_in_docker_container ||= in_docker_container_with_cgroup1? || in_docker_container_with_cgroup2?
89
+ end
90
+
91
+ def in_docker_container_with_cgroup1?
92
+ File.exist?('/proc/1/cgroup') && File.read('/proc/1/cgroup') =~ %r{/docker/}
93
+ end
94
+
95
+ def in_docker_container_with_cgroup2?
96
+ File.exist?('/proc/self/mountinfo') && File.read('/proc/self/mountinfo') =~ %r{/docker/containers/}
97
+ end
98
+
99
+ def default_gw_ips
100
+ default_gw_ips = ['172.17.0.1']
101
+
102
+ if File.exist?('/proc/net/route')
103
+ File.open('/proc/net/route').each_line do |line|
104
+ fields = line.strip.split
105
+ next if fields.size != 11
106
+
107
+ # Destination == 0.0.0.0 and Flags & RTF_GATEWAY != 0
108
+ if fields[1] == '00000000' && (fields[3].hex & 0x2) != 0
109
+ default_gw_ips << IPAddr.new_ntoh([fields[2].hex].pack('L')).to_s
110
+ end
111
+ end
112
+ end
113
+
114
+ default_gw_ips
89
115
  end
90
116
  end
91
117
  end
@@ -5,14 +5,12 @@ require 'active_support/notifications'
5
5
 
6
6
  module Aws
7
7
  module Rails
8
-
9
8
  # Instruments client operation calls for ActiveSupport::Notifications
10
9
  # Each client operation will produce an event with name:
11
10
  # <operation>.<service>.aws
12
11
  # @api private
13
12
  class Notifications < Seahorse::Client::Plugin
14
-
15
- def add_handlers(handlers, config)
13
+ def add_handlers(handlers, _config)
16
14
  # This plugin needs to be first
17
15
  # which means it is called first in the stack, to start recording time,
18
16
  # and returns last
@@ -20,7 +18,6 @@ module Aws
20
18
  end
21
19
 
22
20
  class Handler < Seahorse::Client::Handler
23
-
24
21
  def call(context)
25
22
  event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
26
23
  ActiveSupport::Notifications.instrument(event_name, context: context) do
@@ -10,6 +10,7 @@ module Aws
10
10
  # Initialization Actions
11
11
  Aws::Rails.use_rails_encrypted_credentials
12
12
  Aws::Rails.add_action_mailer_delivery_method
13
+ Aws::Rails.add_action_mailer_delivery_method(:sesv2)
13
14
  Aws::Rails.log_to_rails_logger
14
15
  end
15
16
 
@@ -28,11 +29,15 @@ module Aws
28
29
  #
29
30
  # @param [Symbol] name The name of the ActionMailer delivery method to
30
31
  # register.
31
- # @param [Hash] options The options you wish to pass on to the
32
- # Aws::SES::Client initialization method.
33
- def self.add_action_mailer_delivery_method(name = :ses, options = {})
32
+ # @param [Hash] client_options The options you wish to pass on to the
33
+ # Aws::SES[V2]::Client initialization method.
34
+ def self.add_action_mailer_delivery_method(name = :ses, client_options = {})
34
35
  ActiveSupport.on_load(:action_mailer) do
35
- add_delivery_method(name, Aws::Rails::Mailer, options)
36
+ if name == :sesv2
37
+ add_delivery_method(name, Aws::Rails::Sesv2Mailer, client_options)
38
+ else
39
+ add_delivery_method(name, Aws::Rails::SesMailer, client_options)
40
+ end
36
41
  end
37
42
  end
38
43
 
@@ -15,25 +15,23 @@ module Aws
15
15
  #
16
16
  # Uses the AWS SDK for Ruby's credential provider chain when creating an SES
17
17
  # client instance.
18
- class Mailer
18
+ class SesMailer
19
19
  # @param [Hash] options Passes along initialization options to
20
20
  # [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
21
21
  def initialize(options = {})
22
22
  @client = SES::Client.new(options)
23
+ @client.config.user_agent_frameworks << 'aws-sdk-rails'
23
24
  end
24
25
 
25
26
  # Rails expects this method to exist, and to handle a Mail::Message object
26
27
  # correctly. Called during mail delivery.
27
28
  def deliver!(message)
28
- send_opts = {}
29
- send_opts[:raw_message] = {}
30
- send_opts[:raw_message][:data] = message.to_s
31
-
32
- if message.respond_to?(:destinations)
33
- send_opts[:destinations] = message.destinations
34
- end
35
-
36
- @client.send_raw_email(send_opts).tap do |response|
29
+ params = {
30
+ raw_message: { data: message.to_s },
31
+ source: message.smtp_envelope_from, # defaults to From header
32
+ destinations: message.smtp_envelope_to # defaults to destinations (To,Cc,Bcc)
33
+ }
34
+ @client.send_raw_email(params).tap do |response|
37
35
  message.header[:ses_message_id] = response.message_id
38
36
  end
39
37
  end
@@ -45,3 +43,7 @@ module Aws
45
43
  end
46
44
  end
47
45
  end
46
+
47
+ # This is for backwards compatibility after introducing support for SESv2.
48
+ # The old mailer is now replaced with the new SES (v1) mailer.
49
+ Aws::Rails::Mailer = Aws::Rails::SesMailer
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sesv2'
4
+
5
+ module Aws
6
+ module Rails
7
+ # Provides a delivery method for ActionMailer that uses Amazon Simple Email
8
+ # Service V2.
9
+ #
10
+ # Once you have an SESv2 delivery method you can configure Rails to
11
+ # use this for ActionMailer in your environment configuration
12
+ # (e.g. RAILS_ROOT/config/environments/production.rb)
13
+ #
14
+ # config.action_mailer.delivery_method = :sesv2
15
+ #
16
+ # Uses the AWS SDK for Ruby's credential provider chain when creating an SESV2
17
+ # client instance.
18
+ class Sesv2Mailer
19
+ # @param [Hash] options Passes along initialization options to
20
+ # [Aws::SESV2::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SESV2/Client.html#initialize-instance_method).
21
+ def initialize(options = {})
22
+ @client = SESV2::Client.new(options)
23
+ @client.config.user_agent_frameworks << 'aws-sdk-rails'
24
+ end
25
+
26
+ # Rails expects this method to exist, and to handle a Mail::Message object
27
+ # correctly. Called during mail delivery.
28
+ def deliver!(message)
29
+ params = { content: { raw: { data: message.to_s } } }
30
+ # smtp_envelope_from will default to the From address *without* sender names.
31
+ # By omitting this param, SESv2 will correctly use sender names from the mail headers.
32
+ # We should only use smtp_envelope_from when it was explicitly set (instance variable set)
33
+ params[:from_email_address] = message.smtp_envelope_from if message.instance_variable_get(:@smtp_envelope_from)
34
+ params[:destination] = {
35
+ to_addresses: to_addresses(message),
36
+ cc_addresses: message.cc,
37
+ bcc_addresses: message.bcc
38
+ }
39
+
40
+ @client.send_email(params).tap do |response|
41
+ message.header[:ses_message_id] = response.message_id
42
+ end
43
+ end
44
+
45
+ # ActionMailer expects this method to be present and to return a hash.
46
+ def settings
47
+ {}
48
+ end
49
+
50
+ private
51
+
52
+ # smtp_envelope_to will default to the full destinations (To, Cc, Bcc)
53
+ # SES v2 API prefers each component split out into a destination hash.
54
+ # When smtp_envelope_to was set, use it explicitly for to_address only.
55
+ def to_addresses(message)
56
+ message.instance_variable_get(:@smtp_envelope_to) ? message.smtp_envelope_to : message.to
57
+ end
58
+ end
59
+ end
60
+ end
@@ -3,7 +3,6 @@
3
3
  module Aws
4
4
  module Rails
5
5
  module SqsActiveJob
6
-
7
6
  # @return [Configuration] the (singleton) Configuration
8
7
  def self.config
9
8
  @config ||= Configuration.new
@@ -21,24 +20,26 @@ module Aws
21
20
  # Configuration for AWS SQS ActiveJob.
22
21
  # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
23
22
  class Configuration
24
-
25
23
  # Default configuration options
26
24
  # @api private
27
25
  DEFAULTS = {
28
- max_messages: 10,
29
- visibility_timeout: 120,
26
+ max_messages: 10,
30
27
  shutdown_timeout: 15,
28
+ retry_standard_errors: true, # TODO: Remove in next MV
31
29
  queues: {},
32
30
  logger: ::Rails.logger,
33
- message_group_id: 'SqsActiveJobGroup'
34
- }
31
+ message_group_id: 'SqsActiveJobGroup',
32
+ excluded_deduplication_keys: ['job_id']
33
+ }.freeze
35
34
 
36
35
  # @api private
37
36
  attr_accessor :queues, :max_messages, :visibility_timeout,
38
37
  :shutdown_timeout, :client, :logger,
39
38
  :async_queue_error_handler, :message_group_id
40
39
 
41
- # Don't use this method directly: Confugration is a singleton class, use
40
+ attr_reader :excluded_deduplication_keys
41
+
42
+ # Don't use this method directly: Configuration is a singleton class, use
42
43
  # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
43
44
  #
44
45
  # @param [Hash] options
@@ -50,6 +51,8 @@ module Aws
50
51
  # The max number of messages to poll for in a batch.
51
52
  #
52
53
  # @option options [Integer] :visibility_timeout
54
+ # If unset, the visibility timeout configured on the
55
+ # SQS queue will be used.
53
56
  # The visibility timeout is the number of seconds
54
57
  # that a message will not be processable by any other consumers.
55
58
  # You should set this value to be longer than your expected job runtime
@@ -62,11 +65,21 @@ module Aws
62
65
  # will not be deleted from the SQS queue and will be retryable after
63
66
  # the visibility timeout.
64
67
  #
68
+ # @ option options [Boolean] :retry_standard_errors
69
+ # If `true`, StandardErrors raised by ActiveJobs are left on the queue
70
+ # and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
71
+ # This behavior overrides the standard Rails ActiveJob
72
+ # [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
73
+ # behavior. When set to `true` the retries provided by this will be
74
+ # on top of any retries configured on the job with `retry_on`.
75
+ # When `false`, retry behavior is fully configured
76
+ # through `retry_on`/`discard_on` on the ActiveJobs.
77
+ #
65
78
  # @option options [ActiveSupport::Logger] :logger Logger to use
66
79
  # for the poller.
67
80
  #
68
81
  # @option options [String] :config_file
69
- # Override file to load configuration from. If not specified will
82
+ # Override file to load configuration from. If not specified will
70
83
  # attempt to load from config/aws_sqs_active_job.yml.
71
84
  #
72
85
  # @option options [String] :message_group_id (SqsActiveJobGroup)
@@ -80,18 +93,31 @@ module Aws
80
93
  # +active_job.queue_adapter = :amazon_sqs_async+. Called with:
81
94
  # [error, job, job_options]
82
95
  #
83
- # @option options [SQS::Client] :client SQS Client to use. A default
96
+ # @option options [SQS::Client] :client SQS Client to use. A default
84
97
  # client will be created if none is provided.
98
+ #
99
+ # @option options [Array] :excluded_deduplication_keys (['job_id'])
100
+ # The type of keys stored in the array should be String or Symbol.
101
+ # Using this option, job_id is implicitly added to the keys.
102
+
85
103
  def initialize(options = {})
86
- options[:config_file] ||= config_file if config_file.exist?
104
+ options[:config_file] ||= config_file if File.exist?(config_file)
87
105
  options = DEFAULTS
88
- .merge(file_options(options))
89
- .merge(options)
106
+ .merge(file_options(options))
107
+ .merge(options)
90
108
  set_attributes(options)
91
109
  end
92
110
 
111
+ def excluded_deduplication_keys=(keys)
112
+ @excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
113
+ end
114
+
93
115
  def client
94
- @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
116
+ @client ||= begin
117
+ client = Aws::SQS::Client.new
118
+ client.config.user_agent_frameworks << 'aws-sdk-rails'
119
+ client
120
+ end
95
121
  end
96
122
 
97
123
  # Return the queue_url for a given job_queue name
@@ -99,7 +125,7 @@ module Aws
99
125
  job_queue = job_queue.to_sym
100
126
  raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
101
127
 
102
- queues[job_queue.to_sym]
128
+ queues[job_queue]
103
129
  end
104
130
 
105
131
  # @api private
@@ -110,9 +136,9 @@ module Aws
110
136
  # @api private
111
137
  def to_h
112
138
  h = {}
113
- self.instance_variables.each do |v|
114
- v_sym = v.to_s.gsub('@', '').to_sym
115
- val = self.instance_variable_get(v)
139
+ instance_variables.each do |v|
140
+ v_sym = v.to_s.delete('@').to_sym
141
+ val = instance_variable_get(v)
116
142
  h[v_sym] = val
117
143
  end
118
144
  h
@@ -122,8 +148,9 @@ module Aws
122
148
 
123
149
  # Set accessible attributes after merged options.
124
150
  def set_attributes(options)
125
- options.keys.each do |opt_name|
151
+ options.each_key do |opt_name|
126
152
  instance_variable_set("@#{opt_name}", options[opt_name])
153
+ client.config.user_agent_frameworks << 'aws-sdk-rails' if opt_name == :client
127
154
  end
128
155
  end
129
156
 
@@ -138,24 +165,34 @@ module Aws
138
165
 
139
166
  def config_file
140
167
  file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
141
- file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
168
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless File.exist?(file)
142
169
  file
143
170
  end
144
171
 
145
172
  # Load options from YAML file
146
173
  def load_from_file(file_path)
147
- require "erb"
148
- opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
174
+ opts = load_yaml(file_path) || {}
149
175
  opts.deep_symbolize_keys
150
176
  end
151
177
 
152
178
  # @return [String] Configuration path found in environment or YAML file.
153
179
  def config_file_path(options)
154
- options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
180
+ options[:config_file] || ENV.fetch('AWS_SQS_ACTIVE_JOB_CONFIG_FILE', nil)
155
181
  end
156
182
 
157
- def user_agent
158
- "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
183
+ def load_yaml(file_path)
184
+ require 'erb'
185
+ source = ERB.new(File.read(file_path)).result
186
+
187
+ # Avoid incompatible changes with Psych 4.0.0
188
+ # https://bugs.ruby-lang.org/issues/17866
189
+ # rubocop:disable Security/YAMLLoad
190
+ begin
191
+ YAML.load(source, aliases: true) || {}
192
+ rescue ArgumentError
193
+ YAML.load(source) || {}
194
+ end
195
+ # rubocop:enable Security/YAMLLoad
159
196
  end
160
197
  end
161
198
  end