aws-sdk-rails 2.0.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 9a5ff066628318dc1cc54e08b560bc7dc82370bc
4
- data.tar.gz: 4c36681e57448fc61714157c9d3e522ded8d7721
2
+ SHA256:
3
+ metadata.gz: 2d4664fce310e44e2613d3caf795eead1a767194d48c7b407854a4a3dbe342f4
4
+ data.tar.gz: c49b2bd9ec55f6b1fc63260c0704bc3e2594d4084742a774a527cbec81dfbad9
5
5
  SHA512:
6
- metadata.gz: 39b54b648b6650b4e66e82c774dabdcf0fa6992148e87f4998442b86bf8deb2d8dfbe2d516dca7cfcf496bb905d9064c5ef10c56e18fb9597a1c446c55a95735
7
- data.tar.gz: 0bd3edda7d062750779b2bd1933555f9dc44a129769f40d2a8a7fc376c493761aee29280cc0addd2804c9ffab1184396c30b82990543f36941ba312bc4025e3e
6
+ metadata.gz: 8b38a5870854a7be9867aeb95e1ae2f3fd0a332289ae1f8124306811e42aab5d0274c2abdb7914ff33d9ccb162c334e06034376d4c4a702ab4b39c56b2667b7e
7
+ data.tar.gz: d82eb9df09e04c9426c3191d994568f3565574c7c2f1ed24d70a0ecd8e288179fb9462480f9795409ac857a5e07839096f4b6fd02ab5545d1926e0fc10aa2964
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 3.3.0
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/aws/rails/sqs_active_job/poller'
4
+
5
+ Aws::Rails::SqsActiveJob::Poller.new.run
@@ -0,0 +1,32 @@
1
+ require 'aws-sessionstore-dynamodb'
2
+
3
+ module ActionDispatch
4
+ module Session
5
+ # Uses the Dynamo DB Session Store implementation to create a class that
6
+ # extends ActionDispatch::Session. Rails will create a :dynamodb_store
7
+ # configuration for session_store from this class name.
8
+ #
9
+ # This class will use the Rails secret_key_base unless otherwise provided.
10
+ #
11
+ # Configuration can also be provided in YAML files from Rails config, either
12
+ # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
13
+ # Configuration files that are environment-specific will take precedence.
14
+ #
15
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
16
+ class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
17
+ def initialize(app, options = {})
18
+ options[:config_file] ||= config_file if config_file.exist?
19
+ options[:secret_key] ||= Rails.application.secret_key_base
20
+ super
21
+ end
22
+
23
+ private
24
+
25
+ def config_file
26
+ file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
27
+ file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
28
+ file
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+
5
+ module ActiveJob
6
+ module QueueAdapters
7
+
8
+ class AmazonSqsAdapter
9
+
10
+ def enqueue(job)
11
+ _enqueue(job)
12
+ end
13
+
14
+ def enqueue_at(job, timestamp)
15
+ delay = (timestamp - Time.now.to_f).floor
16
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
+ _enqueue(job, delay_seconds: delay)
18
+ end
19
+
20
+ private
21
+
22
+ def _enqueue(job, send_message_opts = {})
23
+ body = job.serialize
24
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
+ send_message_opts[:queue_url] = queue_url
26
+ send_message_opts[:message_body] = Aws::Json.dump(body)
27
+ send_message_opts[:message_attributes] = message_attributes(job)
28
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
29
+ end
30
+
31
+ def message_attributes(job)
32
+ {
33
+ 'aws_sqs_active_job_class' => {
34
+ string_value: job.class.to_s,
35
+ data_type: 'String'
36
+ },
37
+ 'aws_sqs_active_job_version' => {
38
+ string_value: Aws::Rails::VERSION,
39
+ data_type: 'String'
40
+ }
41
+ }
42
+ end
43
+ end
44
+
45
+ # create an alias to allow `:amazon` to be used as the adapter name
46
+ # `:amazon` is the convention used for ActionMailer and ActiveStorage
47
+ AmazonAdapter = AmazonSqsAdapter
48
+ end
49
+ end
@@ -1,37 +1,19 @@
1
- require_relative 'aws/rails/mailer'
2
-
3
- module Aws
4
- module Rails
1
+ # frozen_string_literal: true
5
2
 
6
- # @api private
7
- class Railtie < ::Rails::Railtie
8
- initializer "aws-sdk-rails.initialize", before: :load_config_initializers do |app|
9
- # Initialization Actions
10
- Aws::Rails.add_action_mailer_delivery_method
11
- Aws::Rails.log_to_rails_logger
12
- end
13
- end
3
+ require_relative 'aws/rails/mailer'
4
+ require_relative 'aws/rails/railtie'
5
+ require_relative 'aws/rails/notifications'
6
+ require_relative 'aws/rails/sqs_active_job/configuration'
7
+ require_relative 'aws/rails/sqs_active_job/executor'
8
+ require_relative 'aws/rails/sqs_active_job/job_runner'
14
9
 
15
- # This is called automatically from the SDK's Railtie, but if you want to
16
- # manually specify options for building the Aws::SES::Client object, you
17
- # can manually call this method.
18
- #
19
- # @param [Symbol] name The name of the ActionMailer delivery method to
20
- # register.
21
- # @param [Hash] options The options you wish to pass on to the
22
- # Aws::SES::Client initialization method.
23
- def self.add_action_mailer_delivery_method(name = :aws_sdk, options = {})
24
- ActiveSupport.on_load(:action_mailer) do
25
- self.add_delivery_method(name, Aws::Rails::Mailer, options)
26
- end
27
- end
10
+ require_relative 'action_dispatch/session/dynamodb_store'
11
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
28
12
 
29
- # Configures the AWS SDK for Ruby's logger to use the Rails logger.
30
- def self.log_to_rails_logger
31
- Aws.config[:logger] = ::Rails.logger
32
- nil
33
- end
13
+ require_relative 'generators/aws_record/base'
34
14
 
15
+ module Aws
16
+ module Rails
17
+ VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
35
18
  end
36
19
  end
37
-
@@ -1,23 +1,23 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'aws-sdk-ses'
2
4
 
3
5
  module Aws
4
6
  module Rails
5
-
6
7
  # Provides a delivery method for ActionMailer that uses Amazon Simple Email
7
8
  # Service.
8
- #
9
+ #
9
10
  # Once you have an SES delivery method you can configure Rails to
10
11
  # use this for ActionMailer in your environment configuration
11
12
  # (e.g. RAILS_ROOT/config/environments/production.rb)
12
13
  #
13
- # config.action_mailer.delivery_method = :aws_sdk
14
+ # config.action_mailer.delivery_method = :ses
14
15
  #
15
- # Uses the AWS SDK for Ruby V2's credential provider chain when creating an
16
- # SES client instance.
16
+ # Uses the AWS SDK for Ruby's credential provider chain when creating an SES
17
+ # client instance.
17
18
  class Mailer
18
-
19
19
  # @param [Hash] options Passes along initialization options to
20
- # [Aws::SES::Client.new](http://docs.aws.amazon.com/sdkforruby/api/Aws/SES/Client.html#initialize-instance_method).
20
+ # [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
21
21
  def initialize(options = {})
22
22
  @client = SES::Client.new(options)
23
23
  end
@@ -33,15 +33,15 @@ module Aws
33
33
  send_opts[:destinations] = message.destinations
34
34
  end
35
35
 
36
- @client.send_raw_email(send_opts)
37
-
36
+ @client.send_raw_email(send_opts).tap do |response|
37
+ message.header[:ses_message_id] = response.message_id
38
+ end
38
39
  end
39
40
 
40
41
  # ActionMailer expects this method to be present and to return a hash.
41
42
  def settings
42
43
  {}
43
44
  end
44
-
45
45
  end
46
46
  end
47
47
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-core'
4
+ require 'active_support/notifications'
5
+
6
+ module Aws
7
+ module Rails
8
+
9
+ # Instruments client operation calls for ActiveSupport::Notifications
10
+ # Each client operation will produce an event with name:
11
+ # <operation>.<service>.aws
12
+ # @api private
13
+ class Notifications < Seahorse::Client::Plugin
14
+
15
+ def add_handlers(handlers, config)
16
+ # This plugin needs to be first
17
+ # which means it is called first in the stack, to start recording time,
18
+ # and returns last
19
+ handlers.add(Handler, step: :initialize, priority: 99)
20
+ end
21
+
22
+ class Handler < Seahorse::Client::Handler
23
+
24
+ def call(context)
25
+ event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
26
+ ActiveSupport::Notifications.instrument(event_name, context: context) do
27
+ @handler.call(context)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ # Use the Rails namespace.
5
+ module Rails
6
+ # @api private
7
+ class Railtie < ::Rails::Railtie
8
+ initializer 'aws-sdk-rails.initialize',
9
+ before: :load_config_initializers do
10
+ # Initialization Actions
11
+ Aws::Rails.use_rails_encrypted_credentials
12
+ Aws::Rails.add_action_mailer_delivery_method
13
+ Aws::Rails.log_to_rails_logger
14
+ end
15
+
16
+ rake_tasks do
17
+ load 'tasks/dynamo_db/session_store.rake'
18
+ load 'tasks/aws_record/migrate.rake'
19
+ end
20
+ end
21
+
22
+ # This is called automatically from the SDK's Railtie, but can be manually
23
+ # called if you want to specify options for building the Aws::SES::Client.
24
+ #
25
+ # @param [Symbol] name The name of the ActionMailer delivery method to
26
+ # register.
27
+ # @param [Hash] options The options you wish to pass on to the
28
+ # Aws::SES::Client initialization method.
29
+ def self.add_action_mailer_delivery_method(name = :ses, options = {})
30
+ ActiveSupport.on_load(:action_mailer) do
31
+ add_delivery_method(name, Aws::Rails::Mailer, options)
32
+ end
33
+ end
34
+
35
+ # Configures the AWS SDK for Ruby's logger to use the Rails logger.
36
+ def self.log_to_rails_logger
37
+ Aws.config[:logger] = ::Rails.logger
38
+ nil
39
+ end
40
+
41
+ # Configures the AWS SDK with credentials from Rails encrypted credentials.
42
+ def self.use_rails_encrypted_credentials
43
+ # limit the config keys we merge to credentials only
44
+ aws_credential_keys = %i[access_key_id secret_access_key session_token]
45
+
46
+ Aws.config.merge!(
47
+ ::Rails.application
48
+ .try(:credentials)
49
+ .try(:aws)
50
+ .to_h.slice(*aws_credential_keys)
51
+ )
52
+ end
53
+
54
+ # Adds ActiveSupport Notifications instrumentation to AWS SDK
55
+ # client operations. Each operation will produce an event with a name:
56
+ # <operation>.<service>.aws. For example, S3's put_object has an event
57
+ # name of: put_object.S3.aws
58
+ def self.instrument_sdk_operations
59
+ Aws.constants.each do |c|
60
+ m = Aws.const_get(c)
61
+ if m.is_a?(Module) && m.const_defined?(:Client) &&
62
+ m.const_get(:Client).superclass == Seahorse::Client::Base
63
+ m.const_get(:Client).add_plugin(Aws::Rails::Notifications)
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ # @return [Configuration] the (singleton) Configuration
8
+ def self.config
9
+ @config ||= Configuration.new
10
+ end
11
+
12
+ # @yield Configuration
13
+ def self.configure
14
+ yield(config)
15
+ end
16
+
17
+ # Holds configuration for AWS SQS ActiveJob
18
+ # Use the Aws::Rails::SqsActiveJob.config to access.
19
+ class Configuration
20
+
21
+ # Default configuration options
22
+ DEFAULTS = {
23
+ max_messages: 10,
24
+ visibility_timeout: 120,
25
+ shutdown_timeout: 15,
26
+ queues: {},
27
+ logger: ::Rails.logger
28
+ }
29
+
30
+ attr_accessor :queues, :max_messages, :visibility_timeout,
31
+ :shutdown_timeout, :client, :logger
32
+
33
+ # @param [Hash] options
34
+ # @option options [Hash[Symbol, String]] :queues - A mapping between the
35
+ # active job queue name and the SQS Queue URL. Note: multiple active
36
+ # job queues can map to the same SQS Queue URL.
37
+ #
38
+ # @option options [Integer] :max_messages -
39
+ # The max number of messages to poll for in a batch.
40
+ #
41
+ # @option options [Integer] :visibility_timeout -
42
+ # The visibility timeout is the number of seconds
43
+ # that a message will not be processable by any other consumers.
44
+ # You should set this value to be longer than your expected job runtime
45
+ # to prevent other processes from picking up an running job.
46
+ # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
47
+ #
48
+ # @option options [Integer] :shutdown_timeout -
49
+ # the amount of time to wait
50
+ # for a clean shutdown. Jobs that are unable to complete in this time
51
+ # will not be deleted from the SQS queue and will be retryable after
52
+ # the visibility timeout.
53
+ #
54
+ # @option options [ActiveSupport::Logger] :logger - Logger to use
55
+ # for the poller.
56
+ #
57
+ # @option options [String] :config_file -
58
+ # Override file to load configuration from. If not specified will
59
+ # attempt to load from config/aws_sqs_active_job.yml.
60
+ #
61
+ # @option options [SQS::Client] :client - SQS Client to use. A default
62
+ # client will be created if none is provided.
63
+ def initialize(options = {})
64
+ options[:config_file] ||= config_file if config_file.exist?
65
+ options = DEFAULTS
66
+ .merge(file_options(options))
67
+ .merge(options)
68
+ set_attributes(options)
69
+ end
70
+
71
+ def client
72
+ @client ||= Aws::SQS::Client.new
73
+ end
74
+
75
+ # Return the queue_url for a given job_queue name
76
+ def queue_url_for(job_queue)
77
+ job_queue = job_queue.to_sym
78
+ raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
79
+
80
+ queues[job_queue.to_sym]
81
+ end
82
+
83
+ def to_s
84
+ to_h.to_s
85
+ end
86
+
87
+ def to_h
88
+ h = {}
89
+ self.instance_variables.each do |v|
90
+ v_sym = v.to_s.gsub('@', '').to_sym
91
+ val = self.instance_variable_get(v)
92
+ h[v_sym] = val
93
+ end
94
+ h
95
+ end
96
+
97
+ private
98
+
99
+ # Set accessible attributes after merged options.
100
+ def set_attributes(options)
101
+ options.keys.each do |opt_name|
102
+ instance_variable_set("@#{opt_name}", options[opt_name])
103
+ end
104
+ end
105
+
106
+ def file_options(options = {})
107
+ file_path = config_file_path(options)
108
+ if file_path
109
+ load_from_file(file_path)
110
+ else
111
+ {}
112
+ end
113
+ end
114
+
115
+ def config_file
116
+ file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
117
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
118
+ file
119
+ end
120
+
121
+ # Load options from YAML file
122
+ def load_from_file(file_path)
123
+ require "erb"
124
+ opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
125
+ opts.deep_symbolize_keys
126
+ end
127
+
128
+ # @return [String] Configuration path found in environment or YAML file.
129
+ def config_file_path(options)
130
+ options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
131
+ end
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent'
4
+
5
+ module Aws
6
+ module Rails
7
+ module SqsActiveJob
8
+ # CLI runner for polling for SQS ActiveJobs
9
+ class Executor
10
+
11
+ DEFAULTS = {
12
+ min_threads: 0,
13
+ max_threads: Concurrent.processor_count,
14
+ auto_terminate: true,
15
+ idletime: 60, # 1 minute
16
+ max_queue: 2,
17
+ fallback_policy: :caller_runs # slow down the producer thread
18
+ }.freeze
19
+
20
+ def initialize(options = {})
21
+ @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
22
+ @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
23
+ end
24
+
25
+ # TODO: Consider catching the exception and sleeping instead of using :caller_runs
26
+ def execute(message)
27
+ @executor.post(message) do |message|
28
+ begin
29
+ job = JobRunner.new(message)
30
+ @logger.info("Running job: #{job.id}[#{job.class_name}]")
31
+ job.run
32
+ message.delete
33
+ rescue Aws::Json::ParseError => e
34
+ @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
35
+ rescue StandardError => e
36
+ # message will not be deleted and will be retried
37
+ job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
38
+ @logger.info "Error processing job #{job_msg}: #{e}"
39
+ @logger.debug e.backtrace.join("\n")
40
+ end
41
+ end
42
+ end
43
+
44
+ def shutdown(timeout=nil)
45
+ @executor.shutdown
46
+ clean_shutdown = @executor.wait_for_termination(timeout)
47
+ if clean_shutdown
48
+ @logger.info 'Clean shutdown complete. All executing jobs finished.'
49
+ else
50
+ @logger.info "Timeout (#{timeout}) exceeded. Some jobs may not have"\
51
+ " finished cleanly. Unfinished jobs will not be removed from"\
52
+ " the queue and can be ru-run once their visibility timeout"\
53
+ " passes."
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end