aws-sdk-rails 3.0.5 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -0
  3. data/bin/aws_sqs_active_job +5 -0
  4. data/lib/action_dispatch/session/dynamodb_store.rb +32 -0
  5. data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb +61 -0
  6. data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb +38 -0
  7. data/lib/aws-sdk-rails.rb +13 -35
  8. data/lib/aws/rails/mailer.rb +1 -1
  9. data/lib/aws/rails/notifications.rb +33 -0
  10. data/lib/aws/rails/railtie.rb +68 -0
  11. data/lib/aws/rails/sqs_active_job/configuration.rb +163 -0
  12. data/lib/aws/rails/sqs_active_job/executor.rb +58 -0
  13. data/lib/aws/rails/sqs_active_job/job_runner.rb +22 -0
  14. data/lib/aws/rails/sqs_active_job/lambda_handler.rb +66 -0
  15. data/lib/aws/rails/sqs_active_job/poller.rb +136 -0
  16. data/lib/generators/aws_record/base.rb +217 -0
  17. data/lib/generators/aws_record/generated_attribute.rb +129 -0
  18. data/lib/generators/aws_record/model/USAGE +24 -0
  19. data/lib/generators/aws_record/model/model_generator.rb +21 -0
  20. data/lib/generators/aws_record/model/templates/model.rb +48 -0
  21. data/lib/generators/aws_record/model/templates/table_config.rb +18 -0
  22. data/lib/generators/aws_record/secondary_index.rb +60 -0
  23. data/lib/generators/dynamo_db/session_store_migration/USAGE +13 -0
  24. data/lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb +46 -0
  25. data/lib/generators/dynamo_db/session_store_migration/templates/dynamo_db_session_store.yml +70 -0
  26. data/lib/generators/dynamo_db/session_store_migration/templates/session_store_migration.rb +9 -0
  27. data/lib/tasks/aws_record/migrate.rake +12 -0
  28. data/lib/tasks/dynamo_db/session_store.rake +8 -0
  29. metadata +89 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 65b9f52f5df1206d26fba5177b30724fb5f0aa66d7888966f533b42dfbba3187
4
- data.tar.gz: 215f61a38c9be614103f56c71f83593e6844b2edd266c16b277a2f5e8d56b01e
3
+ metadata.gz: 06b6734cbbe89bfbec53e688f583d78fb1a1fed48420c0a6b279d0025c8a65d3
4
+ data.tar.gz: 6a84d18c8fbb5807bb88e1b3c803ed94a72bb6287abc244341dfb1e05a595923
5
5
  SHA512:
6
- metadata.gz: 6f216cd5d536b69c872d1332b89bac9fb086ccfd8fe4abd276b971d7ca4ada38d166e66d91ddfd045de4d1dab429acb497d2ca599d38f5ac2b754760fbdb03fd
7
- data.tar.gz: c36a45d9ff981b78bf0c0dfa86189522a05ff5b9a8c6b3abcc9ab95c73861d785bb81705336fb11eed3c5c58bfe79c9bb6dda9f3bfdc3a4c7cb01a22397d70f2
6
+ metadata.gz: 339b7acd4cdc666589ec5921ac583b1b8ed17101fedf6637af957ede0863b0aebe025333ddbb8fdb1fe9a491d2f82f76b7cde81d62ad1eab6c58ae34605c8ecf
7
+ data.tar.gz: c5d8b881c1f5a9521800bc89dd491266088a03dd98845590e5d498e22538baee8f14a014048a3155adcc411a5ded72bc35e02f456097f1d838a4e3cbff905e02
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 3.5.0
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/aws/rails/sqs_active_job/poller'
4
+
5
+ Aws::Rails::SqsActiveJob::Poller.new.run
@@ -0,0 +1,32 @@
1
+ require 'aws-sessionstore-dynamodb'
2
+
3
+ module ActionDispatch
4
+ module Session
5
+ # Uses the Dynamo DB Session Store implementation to create a class that
6
+ # extends ActionDispatch::Session. Rails will create a :dynamodb_store
7
+ # configuration for session_store from this class name.
8
+ #
9
+ # This class will use the Rails secret_key_base unless otherwise provided.
10
+ #
11
+ # Configuration can also be provided in YAML files from Rails config, either
12
+ # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
13
+ # Configuration files that are environment-specific will take precedence.
14
+ #
15
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
16
+ class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
17
+ def initialize(app, options = {})
18
+ options[:config_file] ||= config_file if config_file.exist?
19
+ options[:secret_key] ||= Rails.application.secret_key_base
20
+ super
21
+ end
22
+
23
+ private
24
+
25
+ def config_file
26
+ file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
27
+ file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
28
+ file
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+
5
+ module ActiveJob
6
+ module QueueAdapters
7
+
8
+ class AmazonSqsAdapter
9
+
10
+ def enqueue(job)
11
+ _enqueue(job)
12
+ end
13
+
14
+ def enqueue_at(job, timestamp, opts={})
15
+ delay = (timestamp - Time.now.to_f).floor
16
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
+ _enqueue(job, delay_seconds: delay)
18
+ end
19
+
20
+ private
21
+
22
+ def _enqueue(job, send_message_opts = {})
23
+ body = job.serialize
24
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
+ send_message_opts[:queue_url] = queue_url
26
+ send_message_opts[:message_body] = Aws::Json.dump(body)
27
+ send_message_opts[:message_attributes] = message_attributes(job)
28
+
29
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
30
+ # job_id is unique per initialization of job
31
+ # Remove it from message dup id to ensure run-once behavior
32
+ # with ActiveJob retries
33
+ send_message_opts[:message_deduplication_id] =
34
+ Digest::SHA256.hexdigest(
35
+ Aws::Json.dump(body.except('job_id'))
36
+ )
37
+
38
+ send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
39
+ end
40
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
41
+ end
42
+
43
+ def message_attributes(job)
44
+ {
45
+ 'aws_sqs_active_job_class' => {
46
+ string_value: job.class.to_s,
47
+ data_type: 'String'
48
+ },
49
+ 'aws_sqs_active_job_version' => {
50
+ string_value: Aws::Rails::VERSION,
51
+ data_type: 'String'
52
+ }
53
+ }
54
+ end
55
+ end
56
+
57
+ # create an alias to allow `:amazon` to be used as the adapter name
58
+ # `:amazon` is the convention used for ActionMailer and ActiveStorage
59
+ AmazonAdapter = AmazonSqsAdapter
60
+ end
61
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+ require 'concurrent'
5
+
6
+ module ActiveJob
7
+ module QueueAdapters
8
+
9
+ # == Async adapter for Amazon SQS ActiveJob
10
+ #
11
+ # This adapter queues jobs asynchronously (ie non-blocking). Error handler can be configured
12
+ # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
13
+ #
14
+ # To use this adapter, set up as:
15
+ #
16
+ # config.active_job.queue_adapter = :amazon_sqs_async
17
+ class AmazonSqsAsyncAdapter < AmazonSqsAdapter
18
+
19
+ private
20
+
21
+ def _enqueue(job, send_message_opts = {})
22
+ # FIFO jobs must be queued in order, so do not queue async
23
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
24
+ if Aws::Rails::SqsActiveJob.fifo?(queue_url)
25
+ super(job, send_message_opts)
26
+ else
27
+ Concurrent::Promise
28
+ .execute { super(job, send_message_opts) }
29
+ .on_error do |e|
30
+ Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
31
+ error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
32
+ error_handler.call(e, job, send_message_opts) if error_handler
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -1,43 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'aws/rails/mailer'
4
+ require_relative 'aws/rails/railtie'
5
+ require_relative 'aws/rails/notifications'
6
+ require_relative 'aws/rails/sqs_active_job/configuration'
7
+ require_relative 'aws/rails/sqs_active_job/executor'
8
+ require_relative 'aws/rails/sqs_active_job/job_runner'
9
+ require_relative 'aws/rails/sqs_active_job/lambda_handler'
4
10
 
5
- module Aws
6
- # Use the Rails namespace.
7
- module Rails
8
- # @api private
9
- class Railtie < ::Rails::Railtie
10
- initializer 'aws-sdk-rails.initialize',
11
- before: :load_config_initializers do
12
- # Initialization Actions
13
- Aws::Rails.use_rails_encrypted_credentials
14
- Aws::Rails.add_action_mailer_delivery_method
15
- Aws::Rails.log_to_rails_logger
16
- end
17
- end
18
-
19
- # This is called automatically from the SDK's Railtie, but can be manually
20
- # called if you want to specify options for building the Aws::SES::Client.
21
- #
22
- # @param [Symbol] name The name of the ActionMailer delivery method to
23
- # register.
24
- # @param [Hash] options The options you wish to pass on to the
25
- # Aws::SES::Client initialization method.
26
- def self.add_action_mailer_delivery_method(name = :ses, options = {})
27
- ActiveSupport.on_load(:action_mailer) do
28
- add_delivery_method(name, Aws::Rails::Mailer, options)
29
- end
30
- end
11
+ require_relative 'action_dispatch/session/dynamodb_store'
12
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
13
+ require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
31
14
 
32
- # Configures the AWS SDK for Ruby's logger to use the Rails logger.
33
- def self.log_to_rails_logger
34
- Aws.config[:logger] = ::Rails.logger
35
- nil
36
- end
15
+ require_relative 'generators/aws_record/base'
37
16
 
38
- # Configures the AWS SDK with credentials from Rails encrypted credentials.
39
- def self.use_rails_encrypted_credentials
40
- Aws.config.merge!(::Rails.application.try(:credentials).try(:aws).to_h)
41
- end
17
+ module Aws
18
+ module Rails
19
+ VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
42
20
  end
43
21
  end
@@ -17,7 +17,7 @@ module Aws
17
17
  # client instance.
18
18
  class Mailer
19
19
  # @param [Hash] options Passes along initialization options to
20
- # [Aws::SES::Client.new](http://docs.aws.amazon.com/sdkforruby/api/Aws/SES/Client.html#initialize-instance_method).
20
+ # [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
21
21
  def initialize(options = {})
22
22
  @client = SES::Client.new(options)
23
23
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-core'
4
+ require 'active_support/notifications'
5
+
6
+ module Aws
7
+ module Rails
8
+
9
+ # Instruments client operation calls for ActiveSupport::Notifications
10
+ # Each client operation will produce an event with name:
11
+ # <operation>.<service>.aws
12
+ # @api private
13
+ class Notifications < Seahorse::Client::Plugin
14
+
15
+ def add_handlers(handlers, config)
16
+ # This plugin needs to be first
17
+ # which means it is called first in the stack, to start recording time,
18
+ # and returns last
19
+ handlers.add(Handler, step: :initialize, priority: 99)
20
+ end
21
+
22
+ class Handler < Seahorse::Client::Handler
23
+
24
+ def call(context)
25
+ event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
26
+ ActiveSupport::Notifications.instrument(event_name, context: context) do
27
+ @handler.call(context)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ # Use the Rails namespace.
5
+ module Rails
6
+ # @api private
7
+ class Railtie < ::Rails::Railtie
8
+ initializer 'aws-sdk-rails.initialize',
9
+ before: :load_config_initializers do
10
+ # Initialization Actions
11
+ Aws::Rails.use_rails_encrypted_credentials
12
+ Aws::Rails.add_action_mailer_delivery_method
13
+ Aws::Rails.log_to_rails_logger
14
+ end
15
+
16
+ rake_tasks do
17
+ load 'tasks/dynamo_db/session_store.rake'
18
+ load 'tasks/aws_record/migrate.rake'
19
+ end
20
+ end
21
+
22
+ # This is called automatically from the SDK's Railtie, but can be manually
23
+ # called if you want to specify options for building the Aws::SES::Client.
24
+ #
25
+ # @param [Symbol] name The name of the ActionMailer delivery method to
26
+ # register.
27
+ # @param [Hash] options The options you wish to pass on to the
28
+ # Aws::SES::Client initialization method.
29
+ def self.add_action_mailer_delivery_method(name = :ses, options = {})
30
+ ActiveSupport.on_load(:action_mailer) do
31
+ add_delivery_method(name, Aws::Rails::Mailer, options)
32
+ end
33
+ end
34
+
35
+ # Configures the AWS SDK for Ruby's logger to use the Rails logger.
36
+ def self.log_to_rails_logger
37
+ Aws.config[:logger] = ::Rails.logger
38
+ nil
39
+ end
40
+
41
+ # Configures the AWS SDK with credentials from Rails encrypted credentials.
42
+ def self.use_rails_encrypted_credentials
43
+ # limit the config keys we merge to credentials only
44
+ aws_credential_keys = %i[access_key_id secret_access_key session_token]
45
+
46
+ Aws.config.merge!(
47
+ ::Rails.application
48
+ .try(:credentials)
49
+ .try(:aws)
50
+ .to_h.slice(*aws_credential_keys)
51
+ )
52
+ end
53
+
54
+ # Adds ActiveSupport Notifications instrumentation to AWS SDK
55
+ # client operations. Each operation will produce an event with a name:
56
+ # <operation>.<service>.aws. For example, S3's put_object has an event
57
+ # name of: put_object.S3.aws
58
+ def self.instrument_sdk_operations
59
+ Aws.constants.each do |c|
60
+ m = Aws.const_get(c)
61
+ if m.is_a?(Module) && m.const_defined?(:Client) &&
62
+ m.const_get(:Client).superclass == Seahorse::Client::Base
63
+ m.const_get(:Client).add_plugin(Aws::Rails::Notifications)
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ # @return [Configuration] the (singleton) Configuration
8
+ def self.config
9
+ @config ||= Configuration.new
10
+ end
11
+
12
+ # @yield Configuration
13
+ def self.configure
14
+ yield(config)
15
+ end
16
+
17
+ def self.fifo?(queue_url)
18
+ queue_url.ends_with? '.fifo'
19
+ end
20
+
21
+ # Configuration for AWS SQS ActiveJob.
22
+ # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
23
+ class Configuration
24
+
25
+ # Default configuration options
26
+ # @api private
27
+ DEFAULTS = {
28
+ max_messages: 10,
29
+ visibility_timeout: 120,
30
+ shutdown_timeout: 15,
31
+ queues: {},
32
+ logger: ::Rails.logger,
33
+ message_group_id: 'SqsActiveJobGroup'
34
+ }
35
+
36
+ # @api private
37
+ attr_accessor :queues, :max_messages, :visibility_timeout,
38
+ :shutdown_timeout, :client, :logger,
39
+ :async_queue_error_handler, :message_group_id
40
+
41
+ # Don't use this method directly: Confugration is a singleton class, use
42
+ # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
43
+ #
44
+ # @param [Hash] options
45
+ # @option options [Hash[Symbol, String]] :queues A mapping between the
46
+ # active job queue name and the SQS Queue URL. Note: multiple active
47
+ # job queues can map to the same SQS Queue URL.
48
+ #
49
+ # @option options [Integer] :max_messages
50
+ # The max number of messages to poll for in a batch.
51
+ #
52
+ # @option options [Integer] :visibility_timeout
53
+ # The visibility timeout is the number of seconds
54
+ # that a message will not be processable by any other consumers.
55
+ # You should set this value to be longer than your expected job runtime
56
+ # to prevent other processes from picking up an running job.
57
+ # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
58
+ #
59
+ # @option options [Integer] :shutdown_timeout
60
+ # the amount of time to wait
61
+ # for a clean shutdown. Jobs that are unable to complete in this time
62
+ # will not be deleted from the SQS queue and will be retryable after
63
+ # the visibility timeout.
64
+ #
65
+ # @option options [ActiveSupport::Logger] :logger Logger to use
66
+ # for the poller.
67
+ #
68
+ # @option options [String] :config_file
69
+ # Override file to load configuration from. If not specified will
70
+ # attempt to load from config/aws_sqs_active_job.yml.
71
+ #
72
+ # @option options [String] :message_group_id (SqsActiveJobGroup)
73
+ # The message_group_id to use for queueing messages on a fifo queues.
74
+ # Applies only to jobs queued on FIFO queues.
75
+ # See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
76
+ #
77
+ # @option options [Callable] :async_queue_error_handler An error handler
78
+ # to be called when the async active job adapter experiances an error
79
+ # queueing a job. Only applies when
80
+ # +active_job.queue_adapter = :amazon_sqs_async+. Called with:
81
+ # [error, job, job_options]
82
+ #
83
+ # @option options [SQS::Client] :client SQS Client to use. A default
84
+ # client will be created if none is provided.
85
+ def initialize(options = {})
86
+ options[:config_file] ||= config_file if config_file.exist?
87
+ options = DEFAULTS
88
+ .merge(file_options(options))
89
+ .merge(options)
90
+ set_attributes(options)
91
+ end
92
+
93
+ def client
94
+ @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
95
+ end
96
+
97
+ # Return the queue_url for a given job_queue name
98
+ def queue_url_for(job_queue)
99
+ job_queue = job_queue.to_sym
100
+ raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
101
+
102
+ queues[job_queue.to_sym]
103
+ end
104
+
105
+ # @api private
106
+ def to_s
107
+ to_h.to_s
108
+ end
109
+
110
+ # @api private
111
+ def to_h
112
+ h = {}
113
+ self.instance_variables.each do |v|
114
+ v_sym = v.to_s.gsub('@', '').to_sym
115
+ val = self.instance_variable_get(v)
116
+ h[v_sym] = val
117
+ end
118
+ h
119
+ end
120
+
121
+ private
122
+
123
+ # Set accessible attributes after merged options.
124
+ def set_attributes(options)
125
+ options.keys.each do |opt_name|
126
+ instance_variable_set("@#{opt_name}", options[opt_name])
127
+ end
128
+ end
129
+
130
+ def file_options(options = {})
131
+ file_path = config_file_path(options)
132
+ if file_path
133
+ load_from_file(file_path)
134
+ else
135
+ {}
136
+ end
137
+ end
138
+
139
+ def config_file
140
+ file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
141
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
142
+ file
143
+ end
144
+
145
+ # Load options from YAML file
146
+ def load_from_file(file_path)
147
+ require "erb"
148
+ opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
149
+ opts.deep_symbolize_keys
150
+ end
151
+
152
+ # @return [String] Configuration path found in environment or YAML file.
153
+ def config_file_path(options)
154
+ options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
155
+ end
156
+
157
+ def user_agent
158
+ "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end