aws-sdk-rails 3.0.5 → 3.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -0
  3. data/bin/aws_sqs_active_job +5 -0
  4. data/lib/action_dispatch/session/dynamodb_store.rb +32 -0
  5. data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +61 -0
  6. data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb +38 -0
  7. data/lib/aws-sdk-rails.rb +13 -35
  8. data/lib/aws/rails/mailer.rb +1 -1
  9. data/lib/aws/rails/notifications.rb +33 -0
  10. data/lib/aws/rails/railtie.rb +68 -0
  11. data/lib/aws/rails/sqs_active_job/configuration.rb +163 -0
  12. data/lib/aws/rails/sqs_active_job/executor.rb +58 -0
  13. data/lib/aws/rails/sqs_active_job/job_runner.rb +22 -0
  14. data/lib/aws/rails/sqs_active_job/lambda_handler.rb +66 -0
  15. data/lib/aws/rails/sqs_active_job/poller.rb +136 -0
  16. data/lib/generators/aws_record/base.rb +217 -0
  17. data/lib/generators/aws_record/generated_attribute.rb +129 -0
  18. data/lib/generators/aws_record/model/USAGE +24 -0
  19. data/lib/generators/aws_record/model/model_generator.rb +21 -0
  20. data/lib/generators/aws_record/model/templates/model.rb +48 -0
  21. data/lib/generators/aws_record/model/templates/table_config.rb +18 -0
  22. data/lib/generators/aws_record/secondary_index.rb +60 -0
  23. data/lib/generators/dynamo_db/session_store_migration/USAGE +13 -0
  24. data/lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb +46 -0
  25. data/lib/generators/dynamo_db/session_store_migration/templates/dynamo_db_session_store.yml +70 -0
  26. data/lib/generators/dynamo_db/session_store_migration/templates/session_store_migration.rb +9 -0
  27. data/lib/tasks/aws_record/migrate.rake +12 -0
  28. data/lib/tasks/dynamo_db/session_store.rake +8 -0
  29. metadata +89 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 65b9f52f5df1206d26fba5177b30724fb5f0aa66d7888966f533b42dfbba3187
4
- data.tar.gz: 215f61a38c9be614103f56c71f83593e6844b2edd266c16b277a2f5e8d56b01e
3
+ metadata.gz: 06b6734cbbe89bfbec53e688f583d78fb1a1fed48420c0a6b279d0025c8a65d3
4
+ data.tar.gz: 6a84d18c8fbb5807bb88e1b3c803ed94a72bb6287abc244341dfb1e05a595923
5
5
  SHA512:
6
- metadata.gz: 6f216cd5d536b69c872d1332b89bac9fb086ccfd8fe4abd276b971d7ca4ada38d166e66d91ddfd045de4d1dab429acb497d2ca599d38f5ac2b754760fbdb03fd
7
- data.tar.gz: c36a45d9ff981b78bf0c0dfa86189522a05ff5b9a8c6b3abcc9ab95c73861d785bb81705336fb11eed3c5c58bfe79c9bb6dda9f3bfdc3a4c7cb01a22397d70f2
6
+ metadata.gz: 339b7acd4cdc666589ec5921ac583b1b8ed17101fedf6637af957ede0863b0aebe025333ddbb8fdb1fe9a491d2f82f76b7cde81d62ad1eab6c58ae34605c8ecf
7
+ data.tar.gz: c5d8b881c1f5a9521800bc89dd491266088a03dd98845590e5d498e22538baee8f14a014048a3155adcc411a5ded72bc35e02f456097f1d838a4e3cbff905e02
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 3.5.0
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/aws/rails/sqs_active_job/poller'
4
+
5
+ Aws::Rails::SqsActiveJob::Poller.new.run
@@ -0,0 +1,32 @@
1
+ require 'aws-sessionstore-dynamodb'
2
+
3
+ module ActionDispatch
4
+ module Session
5
+ # Uses the Dynamo DB Session Store implementation to create a class that
6
+ # extends ActionDispatch::Session. Rails will create a :dynamodb_store
7
+ # configuration for session_store from this class name.
8
+ #
9
+ # This class will use the Rails secret_key_base unless otherwise provided.
10
+ #
11
+ # Configuration can also be provided in YAML files from Rails config, either
12
+ # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
13
+ # Configuration files that are environment-specific will take precedence.
14
+ #
15
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
16
+ class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
17
+ def initialize(app, options = {})
18
+ options[:config_file] ||= config_file if config_file.exist?
19
+ options[:secret_key] ||= Rails.application.secret_key_base
20
+ super
21
+ end
22
+
23
+ private
24
+
25
+ def config_file
26
+ file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
27
+ file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
28
+ file
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+
5
+ module ActiveJob
6
+ module QueueAdapters
7
+
8
+ class AmazonSqsAdapter
9
+
10
+ def enqueue(job)
11
+ _enqueue(job)
12
+ end
13
+
14
+ def enqueue_at(job, timestamp, opts={})
15
+ delay = (timestamp - Time.now.to_f).floor
16
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
+ _enqueue(job, delay_seconds: delay)
18
+ end
19
+
20
+ private
21
+
22
+ def _enqueue(job, send_message_opts = {})
23
+ body = job.serialize
24
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
+ send_message_opts[:queue_url] = queue_url
26
+ send_message_opts[:message_body] = Aws::Json.dump(body)
27
+ send_message_opts[:message_attributes] = message_attributes(job)
28
+
29
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
30
+ # job_id is unique per initialization of job
31
+ # Remove it from message dup id to ensure run-once behavior
32
+ # with ActiveJob retries
33
+ send_message_opts[:message_deduplication_id] =
34
+ Digest::SHA256.hexdigest(
35
+ Aws::Json.dump(body.except('job_id'))
36
+ )
37
+
38
+ send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
39
+ end
40
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
41
+ end
42
+
43
+ def message_attributes(job)
44
+ {
45
+ 'aws_sqs_active_job_class' => {
46
+ string_value: job.class.to_s,
47
+ data_type: 'String'
48
+ },
49
+ 'aws_sqs_active_job_version' => {
50
+ string_value: Aws::Rails::VERSION,
51
+ data_type: 'String'
52
+ }
53
+ }
54
+ end
55
+ end
56
+
57
+ # create an alias to allow `:amazon` to be used as the adapter name
58
+ # `:amazon` is the convention used for ActionMailer and ActiveStorage
59
+ AmazonAdapter = AmazonSqsAdapter
60
+ end
61
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+ require 'concurrent'
5
+
6
+ module ActiveJob
7
+ module QueueAdapters
8
+
9
+ # == Async adapter for Amazon SQS ActiveJob
10
+ #
11
+ # This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
12
+ # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
13
+ #
14
+ # To use this adapter, set up as:
15
+ #
16
+ # config.active_job.queue_adapter = :amazon_sqs_async
17
+ class AmazonSqsAsyncAdapter < AmazonSqsAdapter
18
+
19
+ private
20
+
21
+ def _enqueue(job, send_message_opts = {})
22
+ # FIFO jobs must be queued in order, so do not queue async
23
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
24
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
25
+ super(job, send_message_opts)
26
+ else
27
+ Concurrent::Promise
28
+ .execute { super(job, send_message_opts) }
29
+ .on_error do |e|
30
+ Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
31
+ error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
32
+ error_handler.call(e, job, send_message_opts) if error_handler
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -1,43 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'aws/rails/mailer'
4
+ require_relative 'aws/rails/railtie'
5
+ require_relative 'aws/rails/notifications'
6
+ require_relative 'aws/rails/sqs_active_job/configuration'
7
+ require_relative 'aws/rails/sqs_active_job/executor'
8
+ require_relative 'aws/rails/sqs_active_job/job_runner'
9
+ require_relative 'aws/rails/sqs_active_job/lambda_handler'
4
10
 
5
- module Aws
6
- # Use the Rails namespace.
7
- module Rails
8
- # @api private
9
- class Railtie < ::Rails::Railtie
10
- initializer 'aws-sdk-rails.initialize',
11
- before: :load_config_initializers do
12
- # Initialization Actions
13
- Aws::Rails.use_rails_encrypted_credentials
14
- Aws::Rails.add_action_mailer_delivery_method
15
- Aws::Rails.log_to_rails_logger
16
- end
17
- end
18
-
19
- # This is called automatically from the SDK's Railtie, but can be manually
20
- # called if you want to specify options for building the Aws::SES::Client.
21
- #
22
- # @param [Symbol] name The name of the ActionMailer delivery method to
23
- # register.
24
- # @param [Hash] options The options you wish to pass on to the
25
- # Aws::SES::Client initialization method.
26
- def self.add_action_mailer_delivery_method(name = :ses, options = {})
27
- ActiveSupport.on_load(:action_mailer) do
28
- add_delivery_method(name, Aws::Rails::Mailer, options)
29
- end
30
- end
11
+ require_relative 'action_dispatch/session/dynamodb_store'
12
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
13
+ require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
31
14
 
32
- # Configures the AWS SDK for Ruby's logger to use the Rails logger.
33
- def self.log_to_rails_logger
34
- Aws.config[:logger] = ::Rails.logger
35
- nil
36
- end
15
+ require_relative 'generators/aws_record/base'
37
16
 
38
- # Configures the AWS SDK with credentials from Rails encrypted credentials.
39
- def self.use_rails_encrypted_credentials
40
- Aws.config.merge!(::Rails.application.try(:credentials).try(:aws).to_h)
41
- end
17
+ module Aws
18
+ module Rails
19
+ VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
42
20
  end
43
21
  end
@@ -17,7 +17,7 @@ module Aws
17
17
  # client instance.
18
18
  class Mailer
19
19
  # @param [Hash] options Passes along initialization options to
20
- # [Aws::SES::Client.new](http://docs.aws.amazon.com/sdkforruby/api/Aws/SES/Client.html#initialize-instance_method).
20
+ # [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
21
21
  def initialize(options = {})
22
22
  @client = SES::Client.new(options)
23
23
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-core'
4
+ require 'active_support/notifications'
5
+
6
+ module Aws
7
+ module Rails
8
+
9
+ # Instruments client operation calls for ActiveSupport::Notifications
10
+ # Each client operation will produce an event with name:
11
+ # <operation>.<service>.aws
12
+ # @api private
13
+ class Notifications < Seahorse::Client::Plugin
14
+
15
+ def add_handlers(handlers, config)
16
+ # This plugin needs to be first
17
+ # which means it is called first in the stack, to start recording time,
18
+ # and returns last
19
+ handlers.add(Handler, step: :initialize, priority: 99)
20
+ end
21
+
22
+ class Handler < Seahorse::Client::Handler
23
+
24
+ def call(context)
25
+ event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
26
+ ActiveSupport::Notifications.instrument(event_name, context: context) do
27
+ @handler.call(context)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ # Use the Rails namespace.
5
+ module Rails
6
+ # @api private
7
+ class Railtie < ::Rails::Railtie
8
+ initializer 'aws-sdk-rails.initialize',
9
+ before: :load_config_initializers do
10
+ # Initialization Actions
11
+ Aws::Rails.use_rails_encrypted_credentials
12
+ Aws::Rails.add_action_mailer_delivery_method
13
+ Aws::Rails.log_to_rails_logger
14
+ end
15
+
16
+ rake_tasks do
17
+ load 'tasks/dynamo_db/session_store.rake'
18
+ load 'tasks/aws_record/migrate.rake'
19
+ end
20
+ end
21
+
22
+ # This is called automatically from the SDK's Railtie, but can be manually
23
+ # called if you want to specify options for building the Aws::SES::Client.
24
+ #
25
+ # @param [Symbol] name The name of the ActionMailer delivery method to
26
+ # register.
27
+ # @param [Hash] options The options you wish to pass on to the
28
+ # Aws::SES::Client initialization method.
29
+ def self.add_action_mailer_delivery_method(name = :ses, options = {})
30
+ ActiveSupport.on_load(:action_mailer) do
31
+ add_delivery_method(name, Aws::Rails::Mailer, options)
32
+ end
33
+ end
34
+
35
+ # Configures the AWS SDK for Ruby's logger to use the Rails logger.
36
+ def self.log_to_rails_logger
37
+ Aws.config[:logger] = ::Rails.logger
38
+ nil
39
+ end
40
+
41
+ # Configures the AWS SDK with credentials from Rails encrypted credentials.
42
+ def self.use_rails_encrypted_credentials
43
+ # limit the config keys we merge to credentials only
44
+ aws_credential_keys = %i[access_key_id secret_access_key session_token]
45
+
46
+ Aws.config.merge!(
47
+ ::Rails.application
48
+ .try(:credentials)
49
+ .try(:aws)
50
+ .to_h.slice(*aws_credential_keys)
51
+ )
52
+ end
53
+
54
+ # Adds ActiveSupport Notifications instrumentation to AWS SDK
55
+ # client operations. Each operation will produce an event with a name:
56
+ # <operation>.<service>.aws. For example, S3's put_object has an event
57
+ # name of: put_object.S3.aws
58
+ def self.instrument_sdk_operations
59
+ Aws.constants.each do |c|
60
+ m = Aws.const_get(c)
61
+ if m.is_a?(Module) && m.const_defined?(:Client) &&
62
+ m.const_get(:Client).superclass == Seahorse::Client::Base
63
+ m.const_get(:Client).add_plugin(Aws::Rails::Notifications)
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ # @return [Configuration] the (singleton) Configuration
8
+ def self.config
9
+ @config ||= Configuration.new
10
+ end
11
+
12
+ # @yield Configuration
13
+ def self.configure
14
+ yield(config)
15
+ end
16
+
17
+ def self.fifo?(queue_url)
18
+ queue_url.ends_with? '.fifo'
19
+ end
20
+
21
+ # Configuration for AWS SQS ActiveJob.
22
+ # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
23
+ class Configuration
24
+
25
+ # Default configuration options
26
+ # @api private
27
+ DEFAULTS = {
28
+ max_messages: 10,
29
+ visibility_timeout: 120,
30
+ shutdown_timeout: 15,
31
+ queues: {},
32
+ logger: ::Rails.logger,
33
+ message_group_id: 'SqsActiveJobGroup'
34
+ }
35
+
36
+ # @api private
37
+ attr_accessor :queues, :max_messages, :visibility_timeout,
38
+ :shutdown_timeout, :client, :logger,
39
+ :async_queue_error_handler, :message_group_id
40
+
41
+ # Don't use this method directly: Confugration is a singleton class, use
42
+ # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
43
+ #
44
+ # @param [Hash] options
45
+ # @option options [Hash[Symbol, String]] :queues A mapping between the
46
+ # active job queue name and the SQS Queue URL. Note: multiple active
47
+ # job queues can map to the same SQS Queue URL.
48
+ #
49
+ # @option options [Integer] :max_messages
50
+ # The max number of messages to poll for in a batch.
51
+ #
52
+ # @option options [Integer] :visibility_timeout
53
+ # The visibility timeout is the number of seconds
54
+ # that a message will not be processable by any other consumers.
55
+ # You should set this value to be longer than your expected job runtime
56
+ # to prevent other processes from picking up an running job.
57
+ # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
58
+ #
59
+ # @option options [Integer] :shutdown_timeout
60
+ # the amount of time to wait
61
+ # for a clean shutdown. Jobs that are unable to complete in this time
62
+ # will not be deleted from the SQS queue and will be retryable after
63
+ # the visibility timeout.
64
+ #
65
+ # @option options [ActiveSupport::Logger] :logger Logger to use
66
+ # for the poller.
67
+ #
68
+ # @option options [String] :config_file
69
+ # Override file to load configuration from. If not specified will
70
+ # attempt to load from config/aws_sqs_active_job.yml.
71
+ #
72
+ # @option options [String] :message_group_id (SqsActiveJobGroup)
73
+ # The message_group_id to use for queueing messages on a fifo queues.
74
+ # Applies only to jobs queued on FIFO queues.
75
+ # See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
76
+ #
77
+ # @option options [Callable] :async_queue_error_handler An error handler
78
+ # to be called when the async active job adapter experiances an error
79
+ # queueing a job. Only applies when
80
+ # +active_job.queue_adapter = :amazon_sqs_async+. Called with:
81
+ # [error, job, job_options]
82
+ #
83
+ # @option options [SQS::Client] :client SQS Client to use. A default
84
+ # client will be created if none is provided.
85
+ def initialize(options = {})
86
+ options[:config_file] ||= config_file if config_file.exist?
87
+ options = DEFAULTS
88
+ .merge(file_options(options))
89
+ .merge(options)
90
+ set_attributes(options)
91
+ end
92
+
93
+ def client
94
+ @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
95
+ end
96
+
97
+ # Return the queue_url for a given job_queue name
98
+ def queue_url_for(job_queue)
99
+ job_queue = job_queue.to_sym
100
+ raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
101
+
102
+ queues[job_queue.to_sym]
103
+ end
104
+
105
+ # @api private
106
+ def to_s
107
+ to_h.to_s
108
+ end
109
+
110
+ # @api private
111
+ def to_h
112
+ h = {}
113
+ self.instance_variables.each do |v|
114
+ v_sym = v.to_s.gsub('@', '').to_sym
115
+ val = self.instance_variable_get(v)
116
+ h[v_sym] = val
117
+ end
118
+ h
119
+ end
120
+
121
+ private
122
+
123
+ # Set accessible attributes after merged options.
124
+ def set_attributes(options)
125
+ options.keys.each do |opt_name|
126
+ instance_variable_set("@#{opt_name}", options[opt_name])
127
+ end
128
+ end
129
+
130
+ def file_options(options = {})
131
+ file_path = config_file_path(options)
132
+ if file_path
133
+ load_from_file(file_path)
134
+ else
135
+ {}
136
+ end
137
+ end
138
+
139
+ def config_file
140
+ file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
141
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
142
+ file
143
+ end
144
+
145
+ # Load options from YAML file
146
+ def load_from_file(file_path)
147
+ require "erb"
148
+ opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
149
+ opts.deep_symbolize_keys
150
+ end
151
+
152
+ # @return [String] Configuration path found in environment or YAML file.
153
+ def config_file_path(options)
154
+ options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
155
+ end
156
+
157
+ def user_agent
158
+ "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end