aws-sdk-rails 3.2.1 → 3.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b16c1ca75c59341fdc3696d91648fda6901937f31b5b4c55b5d1ccf3bcc9af47
4
- data.tar.gz: 9c9c22d4740ea76b06fa7ee6b8d525450fc982e3c73db0fe81234356dbd71a86
3
+ metadata.gz: 839c723ba30e8acd11e135351278a0422b4f7e6a09a364b8a3fda7d33aa2e899
4
+ data.tar.gz: af1ba6b6525f17e6a46cc16aa4ccbaaf8317860ddf5cd6128d3883c82660ee24
5
5
  SHA512:
6
- metadata.gz: 2ef9e3bf5bde8d91b25635e2bed4028a163547f6e8eabb3ece2e08d1482e6edce16386a916240f344dcce27bc8efec3c5c424236fed6f150e4a6912165648914
7
- data.tar.gz: 0052b3633ce5fc655f4b6fdd80851049196e1c7c7a7d6cc92e17d52fc6fcc8ae69712bc411d3f0feb369dfd835e23912c67802ff32abe3654cf3b4feee87c02a
6
+ metadata.gz: 8288a0c8551bc30cac782c7510a825fdcd9d8ba31bd1d2feda8c2a20ad29f569194237f686f191f1a57ee12eb91a08bb1c5b65b07519af5b62bd7f56ab3f79cb
7
+ data.tar.gz: d6835fc12e6a4a5bc9573b4c5b997a1df5d17e49c28ea2ea16e76f8ebc7b2c8c515026b0a48fb0b15eaab3ba22f1904782f553a03a784dcd665d71723ddd47bd
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 3.6.1
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/aws/rails/sqs_active_job/poller'
4
+
5
+ Aws::Rails::SqsActiveJob::Poller.new.run
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+
5
+ module ActiveJob
6
+ module QueueAdapters
7
+
8
+ class AmazonSqsAdapter
9
+
10
+ def enqueue(job)
11
+ _enqueue(job)
12
+ end
13
+
14
+ def enqueue_at(job, timestamp, opts={})
15
+ delay = (timestamp - Time.now.to_f).floor
16
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
+ _enqueue(job, delay_seconds: delay)
18
+ end
19
+
20
+ private
21
+
22
+ def _enqueue(job, send_message_opts = {})
23
+ body = job.serialize
24
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
+ send_message_opts[:queue_url] = queue_url
26
+ send_message_opts[:message_body] = Aws::Json.dump(body)
27
+ send_message_opts[:message_attributes] = message_attributes(job)
28
+
29
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
30
+ # job_id is unique per initialization of job
31
+ # Remove it from message dup id to ensure run-once behavior
32
+ # with ActiveJob retries
33
+ send_message_opts[:message_deduplication_id] =
34
+ Digest::SHA256.hexdigest(
35
+ Aws::Json.dump(body.except('job_id'))
36
+ )
37
+
38
+ send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
39
+ end
40
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
41
+ end
42
+
43
+ def message_attributes(job)
44
+ {
45
+ 'aws_sqs_active_job_class' => {
46
+ string_value: job.class.to_s,
47
+ data_type: 'String'
48
+ },
49
+ 'aws_sqs_active_job_version' => {
50
+ string_value: Aws::Rails::VERSION,
51
+ data_type: 'String'
52
+ }
53
+ }
54
+ end
55
+ end
56
+
57
+ # create an alias to allow `:amazon` to be used as the adapter name
58
+ # `:amazon` is the convention used for ActionMailer and ActiveStorage
59
+ AmazonAdapter = AmazonSqsAdapter
60
+ end
61
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+ require 'concurrent'
5
+
6
+ module ActiveJob
7
+ module QueueAdapters
8
+
9
+ # == Async adapter for Amazon SQS ActiveJob
10
+ #
11
+ # This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
12
+ # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
13
+ #
14
+ # To use this adapter, set up as:
15
+ #
16
+ # config.active_job.queue_adapter = :amazon_sqs_async
17
+ class AmazonSqsAsyncAdapter < AmazonSqsAdapter
18
+
19
+ private
20
+
21
+ def _enqueue(job, send_message_opts = {})
22
+ # FIFO jobs must be queued in order, so do not queue async
23
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
24
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
25
+ super(job, send_message_opts)
26
+ else
27
+ Concurrent::Promise
28
+ .execute { super(job, send_message_opts) }
29
+ .on_error do |e|
30
+ Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
31
+ error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
32
+ error_handler.call(e, job, send_message_opts) if error_handler
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
data/lib/aws-sdk-rails.rb CHANGED
@@ -3,5 +3,20 @@
3
3
  require_relative 'aws/rails/mailer'
4
4
  require_relative 'aws/rails/railtie'
5
5
  require_relative 'aws/rails/notifications'
6
+ require_relative 'aws/rails/sqs_active_job/configuration'
7
+ require_relative 'aws/rails/sqs_active_job/executor'
8
+ require_relative 'aws/rails/sqs_active_job/job_runner'
9
+ require_relative 'aws/rails/sqs_active_job/lambda_handler'
10
+ require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
6
11
 
7
12
  require_relative 'action_dispatch/session/dynamodb_store'
13
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
14
+ require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
15
+
16
+ require_relative 'generators/aws_record/base'
17
+
18
+ module Aws
19
+ module Rails
20
+ VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
21
+ end
22
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ # Middleware to handle requests from the SQS Daemon present on Elastic Beanstalk worker environments.
6
+ class EbsSqsActiveJobMiddleware
7
+ INTERNAL_ERROR_MESSAGE = 'Failed to execute job - see Rails log for more details.'
8
+ INTERNAL_ERROR_RESPONSE = [500, { 'Content-Type' => 'text/plain' }, [INTERNAL_ERROR_MESSAGE]].freeze
9
+ FORBIDDEN_MESSAGE = 'Request with aws-sqsd user agent was made from untrusted address.'
10
+ FORBIDDEN_RESPONSE = [403, { 'Content-Type' => 'text/plain' }, [FORBIDDEN_MESSAGE]].freeze
11
+
12
+ def initialize(app)
13
+ @app = app
14
+ @logger = ActiveSupport::Logger.new(STDOUT)
15
+ end
16
+
17
+ def call(env)
18
+ request = ActionDispatch::Request.new(env)
19
+
20
+ # Pass through unless user agent is the SQS Daemon
21
+ return @app.call(env) unless from_sqs_daemon?(request)
22
+
23
+ @logger.debug('aws-rails-sdk middleware detected call from Elastic Beanstalk SQS Daemon.')
24
+
25
+ # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
26
+ unless request.local? || sent_from_docker_host?(request)
27
+ @logger.warn("SQSD request detected from untrusted address #{request.remote_ip}; returning 403 forbidden.")
28
+ return FORBIDDEN_RESPONSE
29
+ end
30
+
31
+ # Execute job or periodic task based on HTTP request context
32
+ periodic_task?(request) ? execute_periodic_task(request) : execute_job(request)
33
+ end
34
+
35
+ private
36
+
37
+ def execute_job(request)
38
+ # Jobs queued from the Active Job SQS adapter contain the JSON message in the request body.
39
+ job = Aws::Json.load(request.body.string)
40
+ job_name = job['job_class']
41
+ @logger.debug("Executing job: #{job_name}")
42
+
43
+ begin
44
+ ActiveJob::Base.execute(job)
45
+ rescue NoMethodError, NameError => e
46
+ @logger.error("Job #{job_name} could not resolve to a class that inherits from Active Job.")
47
+ @logger.error("Error: #{e}")
48
+ return INTERNAL_ERROR_RESPONSE
49
+ end
50
+
51
+ [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran job #{job_name}."]]
52
+ end
53
+
54
+ def execute_periodic_task(request)
55
+ # The beanstalk worker SQS Daemon will add the 'X-Aws-Sqsd-Taskname' for periodic tasks set in cron.yaml.
56
+ job_name = request.headers['X-Aws-Sqsd-Taskname']
57
+ @logger.debug("Creating and executing periodic task: #{job_name}")
58
+
59
+ begin
60
+ job = job_name.constantize.new
61
+ job.perform_now
62
+ rescue NoMethodError, NameError => e
63
+ @logger.error("Periodic task #{job_name} could not resolve to an Active Job class - check the spelling in cron.yaml.")
64
+ @logger.error("Error: #{e}.")
65
+ return INTERNAL_ERROR_RESPONSE
66
+ end
67
+
68
+ [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran periodic task #{job_name}."]]
69
+ end
70
+
71
+ # The beanstalk worker SQS Daemon sets a specific User-Agent headers that begins with 'aws-sqsd'.
72
+ def from_sqs_daemon?(request)
73
+ current_user_agent = request.headers['User-Agent']
74
+
75
+ !current_user_agent.nil? && current_user_agent.start_with?('aws-sqsd')
76
+ end
77
+
78
+ # The beanstalk worker SQS Daemon will add the custom 'X-Aws-Sqsd-Taskname' header for periodic tasks set in cron.yaml.
79
+ def periodic_task?(request)
80
+ !request.headers['X-Aws-Sqsd-Taskname'].nil? && request.headers['X-Aws-Sqsd-Taskname'].present?
81
+ end
82
+
83
+ def sent_from_docker_host?(request)
84
+ app_runs_in_docker_container? && request.remote_ip == '172.17.0.1'
85
+ end
86
+
87
+ def app_runs_in_docker_container?
88
+ @app_runs_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
89
+ end
90
+ end
91
+ end
92
+ end
@@ -13,8 +13,13 @@ module Aws
13
13
  Aws::Rails.log_to_rails_logger
14
14
  end
15
15
 
16
+ initializer 'aws-sdk-rails.insert_middleware' do |app|
17
+ Aws::Rails.add_sqsd_middleware(app)
18
+ end
19
+
16
20
  rake_tasks do
17
21
  load 'tasks/dynamo_db/session_store.rake'
22
+ load 'tasks/aws_record/migrate.rake'
18
23
  end
19
24
  end
20
25
 
@@ -43,10 +48,7 @@ module Aws
43
48
  aws_credential_keys = %i[access_key_id secret_access_key session_token]
44
49
 
45
50
  Aws.config.merge!(
46
- ::Rails.application
47
- .try(:credentials)
48
- .try(:aws)
49
- .to_h.slice(*aws_credential_keys)
51
+ ::Rails.application.credentials[:aws].to_h.slice(*aws_credential_keys)
50
52
  )
51
53
  end
52
54
 
@@ -63,5 +65,21 @@ module Aws
63
65
  end
64
66
  end
65
67
  end
68
+
69
+ # Register a middleware that will handle requests from the Elastic Beanstalk worker SQS Daemon.
70
+ # This will only be added in the presence of the AWS_PROCESS_BEANSTALK_WORKER_REQUESTS environment variable.
71
+ # The expectation is this variable should only be set on EB worker environments.
72
+ def self.add_sqsd_middleware(app)
73
+ is_eb_worker_hosted = Aws::Util.str_2_bool(ENV['AWS_PROCESS_BEANSTALK_WORKER_REQUESTS'].to_s.downcase)
74
+
75
+ return unless is_eb_worker_hosted
76
+
77
+ if app.config.force_ssl
78
+ # SQS Daemon sends requests over HTTP - allow and process them before enforcing SSL.
79
+ app.config.middleware.insert_before(ActionDispatch::SSL, Aws::Rails::EbsSqsActiveJobMiddleware)
80
+ else
81
+ app.config.middleware.use(Aws::Rails::EbsSqsActiveJobMiddleware)
82
+ end
83
+ end
66
84
  end
67
85
  end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ # @return [Configuration] the (singleton) Configuration
8
+ def self.config
9
+ @config ||= Configuration.new
10
+ end
11
+
12
+ # @yield Configuration
13
+ def self.configure
14
+ yield(config)
15
+ end
16
+
17
+ def self.fifo?(queue_url)
18
+ queue_url.ends_with? '.fifo'
19
+ end
20
+
21
+ # Configuration for AWS SQS ActiveJob.
22
+ # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
23
+ class Configuration
24
+
25
+ # Default configuration options
26
+ # @api private
27
+ DEFAULTS = {
28
+ max_messages: 10,
29
+ visibility_timeout: 120,
30
+ shutdown_timeout: 15,
31
+ queues: {},
32
+ logger: ::Rails.logger,
33
+ message_group_id: 'SqsActiveJobGroup'
34
+ }
35
+
36
+ # @api private
37
+ attr_accessor :queues, :max_messages, :visibility_timeout,
38
+ :shutdown_timeout, :client, :logger,
39
+ :async_queue_error_handler, :message_group_id
40
+
41
+ # Don't use this method directly: Confugration is a singleton class, use
42
+ # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
43
+ #
44
+ # @param [Hash] options
45
+ # @option options [Hash[Symbol, String]] :queues A mapping between the
46
+ # active job queue name and the SQS Queue URL. Note: multiple active
47
+ # job queues can map to the same SQS Queue URL.
48
+ #
49
+ # @option options [Integer] :max_messages
50
+ # The max number of messages to poll for in a batch.
51
+ #
52
+ # @option options [Integer] :visibility_timeout
53
+ # The visibility timeout is the number of seconds
54
+ # that a message will not be processable by any other consumers.
55
+ # You should set this value to be longer than your expected job runtime
56
+ # to prevent other processes from picking up an running job.
57
+ # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
58
+ #
59
+ # @option options [Integer] :shutdown_timeout
60
+ # the amount of time to wait
61
+ # for a clean shutdown. Jobs that are unable to complete in this time
62
+ # will not be deleted from the SQS queue and will be retryable after
63
+ # the visibility timeout.
64
+ #
65
+ # @option options [ActiveSupport::Logger] :logger Logger to use
66
+ # for the poller.
67
+ #
68
+ # @option options [String] :config_file
69
+ # Override file to load configuration from. If not specified will
70
+ # attempt to load from config/aws_sqs_active_job.yml.
71
+ #
72
+ # @option options [String] :message_group_id (SqsActiveJobGroup)
73
+ # The message_group_id to use for queueing messages on a fifo queues.
74
+ # Applies only to jobs queued on FIFO queues.
75
+ # See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
76
+ #
77
+ # @option options [Callable] :async_queue_error_handler An error handler
78
+ # to be called when the async active job adapter experiances an error
79
+ # queueing a job. Only applies when
80
+ # +active_job.queue_adapter = :amazon_sqs_async+. Called with:
81
+ # [error, job, job_options]
82
+ #
83
+ # @option options [SQS::Client] :client SQS Client to use. A default
84
+ # client will be created if none is provided.
85
+ def initialize(options = {})
86
+ options[:config_file] ||= config_file if config_file.exist?
87
+ options = DEFAULTS
88
+ .merge(file_options(options))
89
+ .merge(options)
90
+ set_attributes(options)
91
+ end
92
+
93
+ def client
94
+ @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
95
+ end
96
+
97
+ # Return the queue_url for a given job_queue name
98
+ def queue_url_for(job_queue)
99
+ job_queue = job_queue.to_sym
100
+ raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
101
+
102
+ queues[job_queue.to_sym]
103
+ end
104
+
105
+ # @api private
106
+ def to_s
107
+ to_h.to_s
108
+ end
109
+
110
+ # @api private
111
+ def to_h
112
+ h = {}
113
+ self.instance_variables.each do |v|
114
+ v_sym = v.to_s.gsub('@', '').to_sym
115
+ val = self.instance_variable_get(v)
116
+ h[v_sym] = val
117
+ end
118
+ h
119
+ end
120
+
121
+ private
122
+
123
+ # Set accessible attributes after merged options.
124
+ def set_attributes(options)
125
+ options.keys.each do |opt_name|
126
+ instance_variable_set("@#{opt_name}", options[opt_name])
127
+ end
128
+ end
129
+
130
+ def file_options(options = {})
131
+ file_path = config_file_path(options)
132
+ if file_path
133
+ load_from_file(file_path)
134
+ else
135
+ {}
136
+ end
137
+ end
138
+
139
+ def config_file
140
+ file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
141
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
142
+ file
143
+ end
144
+
145
+ # Load options from YAML file
146
+ def load_from_file(file_path)
147
+ require "erb"
148
+ opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
149
+ opts.deep_symbolize_keys
150
+ end
151
+
152
+ # @return [String] Configuration path found in environment or YAML file.
153
+ def config_file_path(options)
154
+ options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
155
+ end
156
+
157
+ def user_agent
158
+ "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end