aws-sdk-rails 3.6.1 → 3.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/bin/aws_sqs_active_job +1 -0
- data/lib/action_dispatch/session/dynamodb_store.rb +9 -3
- data/lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb +78 -0
- data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +33 -37
- data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb +12 -11
- data/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb +31 -5
- data/lib/aws/rails/notifications.rb +1 -4
- data/lib/aws/rails/railtie.rb +9 -4
- data/lib/aws/rails/{mailer.rb → ses_mailer.rb} +12 -10
- data/lib/aws/rails/sesv2_mailer.rb +60 -0
- data/lib/aws/rails/sqs_active_job/configuration.rb +61 -24
- data/lib/aws/rails/sqs_active_job/deduplication.rb +21 -0
- data/lib/aws/rails/sqs_active_job/executor.rb +47 -28
- data/lib/aws/rails/sqs_active_job/job_runner.rb +5 -1
- data/lib/aws/rails/sqs_active_job/lambda_handler.rb +3 -6
- data/lib/aws/rails/sqs_active_job/poller.rb +56 -32
- data/lib/aws-sdk-rails.rb +4 -1
- data/lib/generators/aws_record/base.rb +164 -168
- data/lib/generators/aws_record/generated_attribute.rb +50 -41
- data/lib/generators/aws_record/model/model_generator.rb +8 -4
- data/lib/generators/aws_record/secondary_index.rb +31 -25
- data/lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb +3 -1
- data/lib/tasks/aws_record/migrate.rake +2 -0
- data/lib/tasks/dynamo_db/session_store.rake +2 -0
- metadata +52 -18
- /data/lib/generators/aws_record/model/templates/{model.rb → model.erb} +0 -0
- /data/lib/generators/aws_record/model/templates/{table_config.rb → table_config.erb} +0 -0
- /data/lib/generators/dynamo_db/session_store_migration/templates/{session_store_migration.rb → session_store_migration.erb} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43528604142d01e2ddcddf37ae598a08c11744aa3a9ccc6757dbc7d1932d3546
|
4
|
+
data.tar.gz: 7977e087f6372fc8326193dad744f908cda1d2174fc6f817e9cfe48b286a5cfb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2ceda136ba4f7077b6608787528f2ae97b9ceb3169d9c67f101105e7e512dc8000cdb3bf09e5f96b786544bdb427eb15cc0df6571ecc544f5b1603c51cc034b8
|
7
|
+
data.tar.gz: 1fdb53b5fdc4a752643deb9e360988983ad939cce6ada4a8ce3bc68122959c44c7ca6f98d55987172c4d559d9d010ec652b482576bf12b4e496193b183aa236e
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.13.0
|
data/bin/aws_sqs_active_job
CHANGED
@@ -1,4 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sessionstore-dynamodb'
|
4
|
+
require 'action_dispatch/middleware/session/abstract_store'
|
2
5
|
|
3
6
|
module ActionDispatch
|
4
7
|
module Session
|
@@ -9,13 +12,16 @@ module ActionDispatch
|
|
9
12
|
# This class will use the Rails secret_key_base unless otherwise provided.
|
10
13
|
#
|
11
14
|
# Configuration can also be provided in YAML files from Rails config, either
|
12
|
-
# in "config/session_store.yml" or "config/session_store
|
15
|
+
# in "config/session_store.yml" or "config/session_store/#\\{Rails.env}.yml".
|
13
16
|
# Configuration files that are environment-specific will take precedence.
|
14
17
|
#
|
15
18
|
# @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
|
16
19
|
class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
|
20
|
+
include StaleSessionCheck
|
21
|
+
include SessionObject
|
22
|
+
|
17
23
|
def initialize(app, options = {})
|
18
|
-
options[:config_file] ||= config_file if
|
24
|
+
options[:config_file] ||= config_file if File.exist?(config_file)
|
19
25
|
options[:secret_key] ||= Rails.application.secret_key_base
|
20
26
|
super
|
21
27
|
end
|
@@ -24,7 +30,7 @@ module ActionDispatch
|
|
24
30
|
|
25
31
|
def config_file
|
26
32
|
file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
|
27
|
-
file = Rails.root.join('config/dynamo_db_session_store.yml') unless
|
33
|
+
file = Rails.root.join('config/dynamo_db_session_store.yml') unless File.exist?(file)
|
28
34
|
file
|
29
35
|
end
|
30
36
|
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveJob
|
4
|
+
module QueueAdapters
|
5
|
+
class AmazonSqsAdapter
|
6
|
+
# == build request parameter of Aws::SQS::Client
|
7
|
+
class Params
|
8
|
+
class << self
|
9
|
+
def assured_delay_seconds(timestamp)
|
10
|
+
delay = (timestamp - Time.now.to_f).floor
|
11
|
+
delay = 0 if delay.negative?
|
12
|
+
raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
|
13
|
+
|
14
|
+
delay
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(job, body)
|
19
|
+
@job = job
|
20
|
+
@body = body || job.serialize
|
21
|
+
end
|
22
|
+
|
23
|
+
def queue_url
|
24
|
+
@queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def entry
|
28
|
+
if Aws::Rails::SqsActiveJob.fifo?(queue_url)
|
29
|
+
default_entry.merge(options_for_fifo)
|
30
|
+
else
|
31
|
+
default_entry
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def default_entry
|
38
|
+
{
|
39
|
+
message_body: Aws::Json.dump(@body),
|
40
|
+
message_attributes: message_attributes
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def message_attributes
|
45
|
+
{
|
46
|
+
'aws_sqs_active_job_class' => {
|
47
|
+
string_value: @job.class.to_s,
|
48
|
+
data_type: 'String'
|
49
|
+
},
|
50
|
+
'aws_sqs_active_job_version' => {
|
51
|
+
string_value: Aws::Rails::VERSION,
|
52
|
+
data_type: 'String'
|
53
|
+
}
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def options_for_fifo
|
58
|
+
options = {}
|
59
|
+
options[:message_deduplication_id] =
|
60
|
+
Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
|
61
|
+
|
62
|
+
message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
|
63
|
+
message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
|
64
|
+
|
65
|
+
options[:message_group_id] = message_group_id
|
66
|
+
options
|
67
|
+
end
|
68
|
+
|
69
|
+
def deduplication_body
|
70
|
+
ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
|
71
|
+
ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
|
72
|
+
|
73
|
+
@body.except(*ex_dedup_keys)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -4,53 +4,49 @@ require 'aws-sdk-sqs'
|
|
4
4
|
|
5
5
|
module ActiveJob
|
6
6
|
module QueueAdapters
|
7
|
-
|
8
7
|
class AmazonSqsAdapter
|
9
|
-
|
10
8
|
def enqueue(job)
|
11
9
|
_enqueue(job)
|
12
10
|
end
|
13
11
|
|
14
|
-
def enqueue_at(job, timestamp
|
15
|
-
delay = (timestamp
|
16
|
-
|
17
|
-
_enqueue(job, delay_seconds: delay)
|
12
|
+
def enqueue_at(job, timestamp)
|
13
|
+
delay = Params.assured_delay_seconds(timestamp)
|
14
|
+
_enqueue(job, nil, delay_seconds: delay)
|
18
15
|
end
|
19
16
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
17
|
+
def enqueue_all(jobs)
|
18
|
+
enqueued_count = 0
|
19
|
+
jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
|
20
|
+
queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
|
21
|
+
base_send_message_opts = { queue_url: queue_url }
|
22
|
+
|
23
|
+
same_queue_jobs.each_slice(10) do |chunk|
|
24
|
+
entries = chunk.map do |job|
|
25
|
+
entry = Params.new(job, nil).entry
|
26
|
+
entry[:id] = job.job_id
|
27
|
+
entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
|
28
|
+
entry
|
29
|
+
end
|
30
|
+
|
31
|
+
send_message_opts = base_send_message_opts.deep_dup
|
32
|
+
send_message_opts[:entries] = entries
|
33
|
+
|
34
|
+
send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
|
35
|
+
enqueued_count += send_message_batch_result.successful.count
|
36
|
+
end
|
39
37
|
end
|
40
|
-
|
38
|
+
enqueued_count
|
41
39
|
end
|
42
40
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
}
|
53
|
-
}
|
41
|
+
private
|
42
|
+
|
43
|
+
def _enqueue(job, body = nil, send_message_opts = {})
|
44
|
+
body ||= job.serialize
|
45
|
+
params = Params.new(job, body)
|
46
|
+
send_message_opts = send_message_opts.merge(params.entry)
|
47
|
+
send_message_opts[:queue_url] = params.queue_url
|
48
|
+
|
49
|
+
Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
|
54
50
|
end
|
55
51
|
end
|
56
52
|
|
@@ -5,7 +5,6 @@ require 'concurrent'
|
|
5
5
|
|
6
6
|
module ActiveJob
|
7
7
|
module QueueAdapters
|
8
|
-
|
9
8
|
# == Async adapter for Amazon SQS ActiveJob
|
10
9
|
#
|
11
10
|
# This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
|
@@ -15,22 +14,24 @@ module ActiveJob
|
|
15
14
|
#
|
16
15
|
# config.active_job.queue_adapter = :amazon_sqs_async
|
17
16
|
class AmazonSqsAsyncAdapter < AmazonSqsAdapter
|
18
|
-
|
19
17
|
private
|
20
18
|
|
21
|
-
def _enqueue(job, send_message_opts = {})
|
19
|
+
def _enqueue(job, body = nil, send_message_opts = {})
|
22
20
|
# FIFO jobs must be queued in order, so do not queue async
|
23
21
|
queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
|
24
22
|
if Aws::Rails::SqsActiveJob.fifo?(queue_url)
|
25
|
-
super
|
23
|
+
super
|
26
24
|
else
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
25
|
+
# Serialize is called here because the job’s locale needs to be
|
26
|
+
# determined in this thread and not in some other thread.
|
27
|
+
body = job.serialize
|
28
|
+
Concurrent::Promises
|
29
|
+
.future { super }
|
30
|
+
.rescue do |e|
|
31
|
+
Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
|
32
|
+
error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
|
33
|
+
error_handler&.call(e, job, send_message_opts)
|
34
|
+
end
|
34
35
|
end
|
35
36
|
end
|
36
37
|
end
|
@@ -11,7 +11,7 @@ module Aws
|
|
11
11
|
|
12
12
|
def initialize(app)
|
13
13
|
@app = app
|
14
|
-
@logger =
|
14
|
+
@logger = ::Rails.logger
|
15
15
|
end
|
16
16
|
|
17
17
|
def call(env)
|
@@ -20,11 +20,11 @@ module Aws
|
|
20
20
|
# Pass through unless user agent is the SQS Daemon
|
21
21
|
return @app.call(env) unless from_sqs_daemon?(request)
|
22
22
|
|
23
|
-
@logger.debug('aws-rails
|
23
|
+
@logger.debug('aws-sdk-rails middleware detected call from Elastic Beanstalk SQS Daemon.')
|
24
24
|
|
25
25
|
# Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
|
26
26
|
unless request.local? || sent_from_docker_host?(request)
|
27
|
-
@logger.warn("SQSD request detected from untrusted address #{request.
|
27
|
+
@logger.warn("SQSD request detected from untrusted address #{request.ip}; returning 403 forbidden.")
|
28
28
|
return FORBIDDEN_RESPONSE
|
29
29
|
end
|
30
30
|
|
@@ -81,11 +81,37 @@ module Aws
|
|
81
81
|
end
|
82
82
|
|
83
83
|
def sent_from_docker_host?(request)
|
84
|
-
app_runs_in_docker_container? && request.
|
84
|
+
app_runs_in_docker_container? && default_gw_ips.include?(request.ip)
|
85
85
|
end
|
86
86
|
|
87
87
|
def app_runs_in_docker_container?
|
88
|
-
@app_runs_in_docker_container ||=
|
88
|
+
@app_runs_in_docker_container ||= in_docker_container_with_cgroup1? || in_docker_container_with_cgroup2?
|
89
|
+
end
|
90
|
+
|
91
|
+
def in_docker_container_with_cgroup1?
|
92
|
+
File.exist?('/proc/1/cgroup') && File.read('/proc/1/cgroup') =~ %r{/docker/}
|
93
|
+
end
|
94
|
+
|
95
|
+
def in_docker_container_with_cgroup2?
|
96
|
+
File.exist?('/proc/self/mountinfo') && File.read('/proc/self/mountinfo') =~ %r{/docker/containers/}
|
97
|
+
end
|
98
|
+
|
99
|
+
def default_gw_ips
|
100
|
+
default_gw_ips = ['172.17.0.1']
|
101
|
+
|
102
|
+
if File.exist?('/proc/net/route')
|
103
|
+
File.open('/proc/net/route').each_line do |line|
|
104
|
+
fields = line.strip.split
|
105
|
+
next if fields.size != 11
|
106
|
+
|
107
|
+
# Destination == 0.0.0.0 and Flags & RTF_GATEWAY != 0
|
108
|
+
if fields[1] == '00000000' && (fields[3].hex & 0x2) != 0
|
109
|
+
default_gw_ips << IPAddr.new_ntoh([fields[2].hex].pack('L')).to_s
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
default_gw_ips
|
89
115
|
end
|
90
116
|
end
|
91
117
|
end
|
@@ -5,14 +5,12 @@ require 'active_support/notifications'
|
|
5
5
|
|
6
6
|
module Aws
|
7
7
|
module Rails
|
8
|
-
|
9
8
|
# Instruments client operation calls for ActiveSupport::Notifications
|
10
9
|
# Each client operation will produce an event with name:
|
11
10
|
# <operation>.<service>.aws
|
12
11
|
# @api private
|
13
12
|
class Notifications < Seahorse::Client::Plugin
|
14
|
-
|
15
|
-
def add_handlers(handlers, config)
|
13
|
+
def add_handlers(handlers, _config)
|
16
14
|
# This plugin needs to be first
|
17
15
|
# which means it is called first in the stack, to start recording time,
|
18
16
|
# and returns last
|
@@ -20,7 +18,6 @@ module Aws
|
|
20
18
|
end
|
21
19
|
|
22
20
|
class Handler < Seahorse::Client::Handler
|
23
|
-
|
24
21
|
def call(context)
|
25
22
|
event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
|
26
23
|
ActiveSupport::Notifications.instrument(event_name, context: context) do
|
data/lib/aws/rails/railtie.rb
CHANGED
@@ -10,6 +10,7 @@ module Aws
|
|
10
10
|
# Initialization Actions
|
11
11
|
Aws::Rails.use_rails_encrypted_credentials
|
12
12
|
Aws::Rails.add_action_mailer_delivery_method
|
13
|
+
Aws::Rails.add_action_mailer_delivery_method(:sesv2)
|
13
14
|
Aws::Rails.log_to_rails_logger
|
14
15
|
end
|
15
16
|
|
@@ -28,11 +29,15 @@ module Aws
|
|
28
29
|
#
|
29
30
|
# @param [Symbol] name The name of the ActionMailer delivery method to
|
30
31
|
# register.
|
31
|
-
# @param [Hash]
|
32
|
-
# Aws::SES::Client initialization method.
|
33
|
-
def self.add_action_mailer_delivery_method(name = :ses,
|
32
|
+
# @param [Hash] client_options The options you wish to pass on to the
|
33
|
+
# Aws::SES[V2]::Client initialization method.
|
34
|
+
def self.add_action_mailer_delivery_method(name = :ses, client_options = {})
|
34
35
|
ActiveSupport.on_load(:action_mailer) do
|
35
|
-
|
36
|
+
if name == :sesv2
|
37
|
+
add_delivery_method(name, Aws::Rails::Sesv2Mailer, client_options)
|
38
|
+
else
|
39
|
+
add_delivery_method(name, Aws::Rails::SesMailer, client_options)
|
40
|
+
end
|
36
41
|
end
|
37
42
|
end
|
38
43
|
|
@@ -15,25 +15,23 @@ module Aws
|
|
15
15
|
#
|
16
16
|
# Uses the AWS SDK for Ruby's credential provider chain when creating an SES
|
17
17
|
# client instance.
|
18
|
-
class
|
18
|
+
class SesMailer
|
19
19
|
# @param [Hash] options Passes along initialization options to
|
20
20
|
# [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
|
21
21
|
def initialize(options = {})
|
22
22
|
@client = SES::Client.new(options)
|
23
|
+
@client.config.user_agent_frameworks << 'aws-sdk-rails'
|
23
24
|
end
|
24
25
|
|
25
26
|
# Rails expects this method to exist, and to handle a Mail::Message object
|
26
27
|
# correctly. Called during mail delivery.
|
27
28
|
def deliver!(message)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
@client.send_raw_email(send_opts).tap do |response|
|
29
|
+
params = {
|
30
|
+
raw_message: { data: message.to_s },
|
31
|
+
source: message.smtp_envelope_from, # defaults to From header
|
32
|
+
destinations: message.smtp_envelope_to # defaults to destinations (To,Cc,Bcc)
|
33
|
+
}
|
34
|
+
@client.send_raw_email(params).tap do |response|
|
37
35
|
message.header[:ses_message_id] = response.message_id
|
38
36
|
end
|
39
37
|
end
|
@@ -45,3 +43,7 @@ module Aws
|
|
45
43
|
end
|
46
44
|
end
|
47
45
|
end
|
46
|
+
|
47
|
+
# This is for backwards compatibility after introducing support for SESv2.
|
48
|
+
# The old mailer is now replaced with the new SES (v1) mailer.
|
49
|
+
Aws::Rails::Mailer = Aws::Rails::SesMailer
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-sesv2'
|
4
|
+
|
5
|
+
module Aws
|
6
|
+
module Rails
|
7
|
+
# Provides a delivery method for ActionMailer that uses Amazon Simple Email
|
8
|
+
# Service V2.
|
9
|
+
#
|
10
|
+
# Once you have an SESv2 delivery method you can configure Rails to
|
11
|
+
# use this for ActionMailer in your environment configuration
|
12
|
+
# (e.g. RAILS_ROOT/config/environments/production.rb)
|
13
|
+
#
|
14
|
+
# config.action_mailer.delivery_method = :sesv2
|
15
|
+
#
|
16
|
+
# Uses the AWS SDK for Ruby's credential provider chain when creating an SESV2
|
17
|
+
# client instance.
|
18
|
+
class Sesv2Mailer
|
19
|
+
# @param [Hash] options Passes along initialization options to
|
20
|
+
# [Aws::SESV2::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SESV2/Client.html#initialize-instance_method).
|
21
|
+
def initialize(options = {})
|
22
|
+
@client = SESV2::Client.new(options)
|
23
|
+
@client.config.user_agent_frameworks << 'aws-sdk-rails'
|
24
|
+
end
|
25
|
+
|
26
|
+
# Rails expects this method to exist, and to handle a Mail::Message object
|
27
|
+
# correctly. Called during mail delivery.
|
28
|
+
def deliver!(message)
|
29
|
+
params = { content: { raw: { data: message.to_s } } }
|
30
|
+
# smtp_envelope_from will default to the From address *without* sender names.
|
31
|
+
# By omitting this param, SESv2 will correctly use sender names from the mail headers.
|
32
|
+
# We should only use smtp_envelope_from when it was explicitly set (instance variable set)
|
33
|
+
params[:from_email_address] = message.smtp_envelope_from if message.instance_variable_get(:@smtp_envelope_from)
|
34
|
+
params[:destination] = {
|
35
|
+
to_addresses: to_addresses(message),
|
36
|
+
cc_addresses: message.cc,
|
37
|
+
bcc_addresses: message.bcc
|
38
|
+
}
|
39
|
+
|
40
|
+
@client.send_email(params).tap do |response|
|
41
|
+
message.header[:ses_message_id] = response.message_id
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# ActionMailer expects this method to be present and to return a hash.
|
46
|
+
def settings
|
47
|
+
{}
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
# smtp_envelope_to will default to the full destinations (To, Cc, Bcc)
|
53
|
+
# SES v2 API prefers each component split out into a destination hash.
|
54
|
+
# When smtp_envelope_to was set, use it explicitly for to_address only.
|
55
|
+
def to_addresses(message)
|
56
|
+
message.instance_variable_get(:@smtp_envelope_to) ? message.smtp_envelope_to : message.to
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -3,7 +3,6 @@
|
|
3
3
|
module Aws
|
4
4
|
module Rails
|
5
5
|
module SqsActiveJob
|
6
|
-
|
7
6
|
# @return [Configuration] the (singleton) Configuration
|
8
7
|
def self.config
|
9
8
|
@config ||= Configuration.new
|
@@ -21,24 +20,26 @@ module Aws
|
|
21
20
|
# Configuration for AWS SQS ActiveJob.
|
22
21
|
# Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
|
23
22
|
class Configuration
|
24
|
-
|
25
23
|
# Default configuration options
|
26
24
|
# @api private
|
27
25
|
DEFAULTS = {
|
28
|
-
max_messages:
|
29
|
-
visibility_timeout: 120,
|
26
|
+
max_messages: 10,
|
30
27
|
shutdown_timeout: 15,
|
28
|
+
retry_standard_errors: true, # TODO: Remove in next MV
|
31
29
|
queues: {},
|
32
30
|
logger: ::Rails.logger,
|
33
|
-
message_group_id: 'SqsActiveJobGroup'
|
34
|
-
|
31
|
+
message_group_id: 'SqsActiveJobGroup',
|
32
|
+
excluded_deduplication_keys: ['job_id']
|
33
|
+
}.freeze
|
35
34
|
|
36
35
|
# @api private
|
37
36
|
attr_accessor :queues, :max_messages, :visibility_timeout,
|
38
37
|
:shutdown_timeout, :client, :logger,
|
39
38
|
:async_queue_error_handler, :message_group_id
|
40
39
|
|
41
|
-
|
40
|
+
attr_reader :excluded_deduplication_keys
|
41
|
+
|
42
|
+
# Don't use this method directly: Configuration is a singleton class, use
|
42
43
|
# +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
|
43
44
|
#
|
44
45
|
# @param [Hash] options
|
@@ -50,6 +51,8 @@ module Aws
|
|
50
51
|
# The max number of messages to poll for in a batch.
|
51
52
|
#
|
52
53
|
# @option options [Integer] :visibility_timeout
|
54
|
+
# If unset, the visibility timeout configured on the
|
55
|
+
# SQS queue will be used.
|
53
56
|
# The visibility timeout is the number of seconds
|
54
57
|
# that a message will not be processable by any other consumers.
|
55
58
|
# You should set this value to be longer than your expected job runtime
|
@@ -62,11 +65,21 @@ module Aws
|
|
62
65
|
# will not be deleted from the SQS queue and will be retryable after
|
63
66
|
# the visibility timeout.
|
64
67
|
#
|
68
|
+
# @ option options [Boolean] :retry_standard_errors
|
69
|
+
# If `true`, StandardErrors raised by ActiveJobs are left on the queue
|
70
|
+
# and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
|
71
|
+
# This behavior overrides the standard Rails ActiveJob
|
72
|
+
# [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
|
73
|
+
# behavior. When set to `true` the retries provided by this will be
|
74
|
+
# on top of any retries configured on the job with `retry_on`.
|
75
|
+
# When `false`, retry behavior is fully configured
|
76
|
+
# through `retry_on`/`discard_on` on the ActiveJobs.
|
77
|
+
#
|
65
78
|
# @option options [ActiveSupport::Logger] :logger Logger to use
|
66
79
|
# for the poller.
|
67
80
|
#
|
68
81
|
# @option options [String] :config_file
|
69
|
-
# Override file to load configuration from.
|
82
|
+
# Override file to load configuration from. If not specified will
|
70
83
|
# attempt to load from config/aws_sqs_active_job.yml.
|
71
84
|
#
|
72
85
|
# @option options [String] :message_group_id (SqsActiveJobGroup)
|
@@ -80,18 +93,31 @@ module Aws
|
|
80
93
|
# +active_job.queue_adapter = :amazon_sqs_async+. Called with:
|
81
94
|
# [error, job, job_options]
|
82
95
|
#
|
83
|
-
# @option options [SQS::Client] :client SQS Client to use.
|
96
|
+
# @option options [SQS::Client] :client SQS Client to use. A default
|
84
97
|
# client will be created if none is provided.
|
98
|
+
#
|
99
|
+
# @option options [Array] :excluded_deduplication_keys (['job_id'])
|
100
|
+
# The type of keys stored in the array should be String or Symbol.
|
101
|
+
# Using this option, job_id is implicitly added to the keys.
|
102
|
+
|
85
103
|
def initialize(options = {})
|
86
|
-
options[:config_file] ||= config_file if
|
104
|
+
options[:config_file] ||= config_file if File.exist?(config_file)
|
87
105
|
options = DEFAULTS
|
88
|
-
|
89
|
-
|
106
|
+
.merge(file_options(options))
|
107
|
+
.merge(options)
|
90
108
|
set_attributes(options)
|
91
109
|
end
|
92
110
|
|
111
|
+
def excluded_deduplication_keys=(keys)
|
112
|
+
@excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
|
113
|
+
end
|
114
|
+
|
93
115
|
def client
|
94
|
-
@client ||=
|
116
|
+
@client ||= begin
|
117
|
+
client = Aws::SQS::Client.new
|
118
|
+
client.config.user_agent_frameworks << 'aws-sdk-rails'
|
119
|
+
client
|
120
|
+
end
|
95
121
|
end
|
96
122
|
|
97
123
|
# Return the queue_url for a given job_queue name
|
@@ -99,7 +125,7 @@ module Aws
|
|
99
125
|
job_queue = job_queue.to_sym
|
100
126
|
raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
|
101
127
|
|
102
|
-
queues[job_queue
|
128
|
+
queues[job_queue]
|
103
129
|
end
|
104
130
|
|
105
131
|
# @api private
|
@@ -110,9 +136,9 @@ module Aws
|
|
110
136
|
# @api private
|
111
137
|
def to_h
|
112
138
|
h = {}
|
113
|
-
|
114
|
-
v_sym = v.to_s.
|
115
|
-
val =
|
139
|
+
instance_variables.each do |v|
|
140
|
+
v_sym = v.to_s.delete('@').to_sym
|
141
|
+
val = instance_variable_get(v)
|
116
142
|
h[v_sym] = val
|
117
143
|
end
|
118
144
|
h
|
@@ -122,8 +148,9 @@ module Aws
|
|
122
148
|
|
123
149
|
# Set accessible attributes after merged options.
|
124
150
|
def set_attributes(options)
|
125
|
-
options.
|
151
|
+
options.each_key do |opt_name|
|
126
152
|
instance_variable_set("@#{opt_name}", options[opt_name])
|
153
|
+
client.config.user_agent_frameworks << 'aws-sdk-rails' if opt_name == :client
|
127
154
|
end
|
128
155
|
end
|
129
156
|
|
@@ -138,24 +165,34 @@ module Aws
|
|
138
165
|
|
139
166
|
def config_file
|
140
167
|
file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
|
141
|
-
file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless
|
168
|
+
file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless File.exist?(file)
|
142
169
|
file
|
143
170
|
end
|
144
171
|
|
145
172
|
# Load options from YAML file
|
146
173
|
def load_from_file(file_path)
|
147
|
-
|
148
|
-
opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
|
174
|
+
opts = load_yaml(file_path) || {}
|
149
175
|
opts.deep_symbolize_keys
|
150
176
|
end
|
151
177
|
|
152
178
|
# @return [String] Configuration path found in environment or YAML file.
|
153
179
|
def config_file_path(options)
|
154
|
-
options[:config_file] || ENV
|
180
|
+
options[:config_file] || ENV.fetch('AWS_SQS_ACTIVE_JOB_CONFIG_FILE', nil)
|
155
181
|
end
|
156
182
|
|
157
|
-
def
|
158
|
-
|
183
|
+
def load_yaml(file_path)
|
184
|
+
require 'erb'
|
185
|
+
source = ERB.new(File.read(file_path)).result
|
186
|
+
|
187
|
+
# Avoid incompatible changes with Psych 4.0.0
|
188
|
+
# https://bugs.ruby-lang.org/issues/17866
|
189
|
+
# rubocop:disable Security/YAMLLoad
|
190
|
+
begin
|
191
|
+
YAML.load(source, aliases: true) || {}
|
192
|
+
rescue ArgumentError
|
193
|
+
YAML.load(source) || {}
|
194
|
+
end
|
195
|
+
# rubocop:enable Security/YAMLLoad
|
159
196
|
end
|
160
197
|
end
|
161
198
|
end
|